added registeration of rolling_update_duration and recreate_group_tim…

…es, and functionality of the latter

Dockerfile

@@ -19,10 +19,7 @@ RUN go mod download
 # Copy the go source
 COPY cmd/main.go cmd/main.go
 COPY api/ api/
-COPY pkg/controllers/ pkg/controllers/
-COPY pkg/cert/ pkg/cert/
-COPY pkg/webhooks/ pkg/webhooks/
-COPY pkg/utils pkg/utils
+COPY pkg/ pkg/
 
 # Build
 # the GOARCH has not a default value to allow the binary be built according to the host where the command

cmd/main.go

@@ -35,6 +35,7 @@ import (
 	leaderworkersetv1 "sigs.k8s.io/lws/api/leaderworkerset/v1"
 	"sigs.k8s.io/lws/pkg/cert"
 	"sigs.k8s.io/lws/pkg/controllers"
+	"sigs.k8s.io/lws/pkg/metrics"
 	"sigs.k8s.io/lws/pkg/webhooks"
 	//+kubebuilder:scaffold:imports
 )
@@ -76,6 +77,8 @@ func main() {
 	kubeConfig.QPS = float32(qps)
 	kubeConfig.Burst = burst
 
+	metrics.Register()
+
 	mgr, err := ctrl.NewManager(kubeConfig, ctrl.Options{
 		Scheme:                 scheme,
 		Metrics:                metricsserver.Options{BindAddress: metricsAddr},

go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/onsi/ginkgo/v2 v2.17.1
 	github.com/onsi/gomega v1.32.0
 	github.com/open-policy-agent/cert-controller v0.10.1
+	github.com/prometheus/client_golang v1.18.0
 	k8s.io/api v0.29.3
 	k8s.io/apiextensions-apiserver v0.29.3
 	k8s.io/apimachinery v0.29.3
@@ -48,7 +49,6 @@ require (
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/prometheus/client_golang v1.18.0 // indirect
 	github.com/prometheus/client_model v0.5.0 // indirect
 	github.com/prometheus/common v0.45.0 // indirect
 	github.com/prometheus/procfs v0.12.0 // indirect

pkg/controllers/pod_controller.go

@@ -39,6 +39,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
 
 	leaderworkerset "sigs.k8s.io/lws/api/leaderworkerset/v1"
+	"sigs.k8s.io/lws/pkg/metrics"
 	acceleratorutils "sigs.k8s.io/lws/pkg/utils/accelerators"
 	podutils "sigs.k8s.io/lws/pkg/utils/pod"
 	statefulsetutils "sigs.k8s.io/lws/pkg/utils/statefulset"
@@ -174,6 +175,7 @@ func (r *PodReconciler) handleRestartPolicy(ctx context.Context, pod corev1.Pod,
 	}); err != nil {
 		return false, err
 	}
+	metrics.RecreatingGroup(leader.Name)
 	return true, nil
 }
 

pkg/metrics/metrics.go

@@ -0,0 +1,57 @@
+/*
+Copyright 2023.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metrics
+
+import (
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"sigs.k8s.io/controller-runtime/pkg/metrics"
+)
+
+var (
+	rollingUpdateDuration = prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Subsystem: "lws",
+			Name:      "rolling_update_duration",
+			Help:      "Duration of rolling updates",
+		}, []string{"hash"},
+	)
+
+	recreateGroupTimes = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Subsystem: "lws",
+			Name:      "recreate_group_times",
+			Help:      "number of times a group has been recreated",
+		}, []string{"leadername"},
+	)
+)
+
+func RollingUpdate(hash string, duration time.Duration) {
+	rollingUpdateDuration.WithLabelValues(hash).Observe(duration.Seconds())
+}
+
+func RecreatingGroup(leaderName string) {
+	recreateGroupTimes.WithLabelValues(leaderName).Inc()
+}
+
+func Register() {
+	metrics.Registry.MustRegister(
+		rollingUpdateDuration,
+		recreateGroupTimes,
+	)
+}