Skip to content

Commit 4d51e57

Browse files
stefanprodanstefanprodan
authored
Merge pull request #58 from fluxcd/metrics
Implement Prometheus instrumentation
2 parents 2a839a1 + 633f50e commit 4d51e57

File tree

6 files changed

+112
-21
lines changed

6 files changed

+112
-21
lines changed

‎controllers/alert_controller.go‎

Lines changed: 42 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,32 +18,38 @@ package controllers
1818

1919
import (
2020
"context"
21+
"fmt"
2122
"strings"
23+
"time"
2224

2325
"github.com/go-logr/logr"
2426
corev1 "k8s.io/api/core/v1"
2527
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2628
"k8s.io/apimachinery/pkg/runtime"
29+
"k8s.io/client-go/tools/reference"
2730
ctrl "sigs.k8s.io/controller-runtime"
2831
"sigs.k8s.io/controller-runtime/pkg/client"
2932

3033
"github.com/fluxcd/pkg/apis/meta"
34+
"github.com/fluxcd/pkg/runtime/metrics"
3135

3236
"github.com/fluxcd/notification-controller/api/v1beta1"
3337
)
3438

3539
// AlertReconciler reconciles a Alert object
3640
type AlertReconciler struct {
3741
client.Client
38-
Log logr.Logger
39-
Scheme *runtime.Scheme
42+
Log logr.Logger
43+
Scheme *runtime.Scheme
44+
MetricsRecorder *metrics.Recorder
4045
}
4146

4247
// +kubebuilder:rbac:groups=notification.toolkit.fluxcd.io,resources=alerts,verbs=get;list;watch;create;update;patch;delete
4348
// +kubebuilder:rbac:groups=notification.toolkit.fluxcd.io,resources=alerts/status,verbs=get;update;patch
4449

4550
func (r *AlertReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
4651
ctx := context.Background()
52+
reconcileStart := time.Now()
4753

4854
var alert v1beta1.Alert
4955
if err := r.Get(ctx, req.NamespacedName, &alert); err != nil {
@@ -52,6 +58,15 @@ func (r *AlertReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
5258

5359
log := r.Log.WithValues("controller", strings.ToLower(alert.Kind), "request", req.NamespacedName)
5460

61+
// record reconciliation duration
62+
if r.MetricsRecorder != nil {
63+
objRef, err := reference.GetReference(r.Scheme, &alert)
64+
if err != nil {
65+
return ctrl.Result{}, err
66+
}
67+
defer r.MetricsRecorder.RecordDuration(*objRef, reconcileStart)
68+
}
69+
5570
init := true
5671
if c := meta.GetCondition(alert.Status.Conditions, meta.ReadyCondition); c != nil {
5772
if c.Status == corev1.ConditionTrue {
@@ -75,6 +90,8 @@ func (r *AlertReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
7590
log.Info("Alert initialised")
7691
}
7792

93+
r.recordReadiness(alert, false)
94+
7895
return ctrl.Result{}, nil
7996
}
8097

@@ -83,3 +100,26 @@ func (r *AlertReconciler) SetupWithManager(mgr ctrl.Manager) error {
83100
For(&v1beta1.Alert{}).
84101
Complete(r)
85102
}
103+
104+
func (r *AlertReconciler) recordReadiness(alert v1beta1.Alert, deleted bool) {
105+
if r.MetricsRecorder == nil {
106+
return
107+
}
108+
109+
objRef, err := reference.GetReference(r.Scheme, &alert)
110+
if err != nil {
111+
r.Log.WithValues(
112+
strings.ToLower(alert.Kind),
113+
fmt.Sprintf("%s/%s", alert.GetNamespace(), alert.GetName()),
114+
).Error(err, "unable to record readiness metric")
115+
return
116+
}
117+
if rc := meta.GetCondition(alert.Status.Conditions, meta.ReadyCondition); rc != nil {
118+
r.MetricsRecorder.RecordCondition(*objRef, *rc, deleted)
119+
} else {
120+
r.MetricsRecorder.RecordCondition(*objRef, meta.Condition{
121+
Type: meta.ReadyCondition,
122+
Status: corev1.ConditionUnknown,
123+
}, deleted)
124+
}
125+
}

‎controllers/provider_controller.go‎

Lines changed: 40 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,32 +18,38 @@ package controllers
1818

1919
import (
2020
"context"
21+
"fmt"
2122
"strings"
23+
"time"
2224

2325
"github.com/go-logr/logr"
2426
corev1 "k8s.io/api/core/v1"
2527
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2628
"k8s.io/apimachinery/pkg/runtime"
29+
"k8s.io/client-go/tools/reference"
2730
ctrl "sigs.k8s.io/controller-runtime"
2831
"sigs.k8s.io/controller-runtime/pkg/client"
2932

3033
"github.com/fluxcd/pkg/apis/meta"
34+
"github.com/fluxcd/pkg/runtime/metrics"
3135

3236
"github.com/fluxcd/notification-controller/api/v1beta1"
3337
)
3438

3539
// ProviderReconciler reconciles a Provider object
3640
type ProviderReconciler struct {
3741
client.Client
38-
Log logr.Logger
39-
Scheme *runtime.Scheme
42+
Log logr.Logger
43+
Scheme *runtime.Scheme
44+
MetricsRecorder *metrics.Recorder
4045
}
4146

4247
// +kubebuilder:rbac:groups=notification.toolkit.fluxcd.io,resources=providers,verbs=get;list;watch;create;update;patch;delete
4348
// +kubebuilder:rbac:groups=notification.toolkit.fluxcd.io,resources=providers/status,verbs=get;update;patch
4449

4550
func (r *ProviderReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
4651
ctx := context.Background()
52+
reconcileStart := time.Now()
4753

4854
var provider v1beta1.Provider
4955
if err := r.Get(ctx, req.NamespacedName, &provider); err != nil {
@@ -52,6 +58,15 @@ func (r *ProviderReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
5258

5359
log := r.Log.WithValues("controller", strings.ToLower(provider.Kind), "request", req.NamespacedName)
5460

61+
// record reconciliation duration
62+
if r.MetricsRecorder != nil {
63+
objRef, err := reference.GetReference(r.Scheme, &provider)
64+
if err != nil {
65+
return ctrl.Result{}, err
66+
}
67+
defer r.MetricsRecorder.RecordDuration(*objRef, reconcileStart)
68+
}
69+
5570
init := true
5671
if c := meta.GetCondition(provider.Status.Conditions, meta.ReadyCondition); c != nil {
5772
if c.Status == corev1.ConditionTrue {
@@ -83,3 +98,26 @@ func (r *ProviderReconciler) SetupWithManager(mgr ctrl.Manager) error {
8398
For(&v1beta1.Provider{}).
8499
Complete(r)
85100
}
101+
102+
func (r *ProviderReconciler) recordReadiness(provider v1beta1.Provider, deleted bool) {
103+
if r.MetricsRecorder == nil {
104+
return
105+
}
106+
107+
objRef, err := reference.GetReference(r.Scheme, &provider)
108+
if err != nil {
109+
r.Log.WithValues(
110+
strings.ToLower(provider.Kind),
111+
fmt.Sprintf("%s/%s", provider.GetNamespace(), provider.GetName()),
112+
).Error(err, "unable to record readiness metric")
113+
return
114+
}
115+
if rc := meta.GetCondition(provider.Status.Conditions, meta.ReadyCondition); rc != nil {
116+
r.MetricsRecorder.RecordCondition(*objRef, *rc, deleted)
117+
} else {
118+
r.MetricsRecorder.RecordCondition(*objRef, meta.Condition{
119+
Type: meta.ReadyCondition,
120+
Status: corev1.ConditionUnknown,
121+
}, deleted)
122+
}
123+
}

‎controllers/receiver_controller.go‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,15 +30,17 @@ import (
3030
"sigs.k8s.io/controller-runtime/pkg/client"
3131

3232
"github.com/fluxcd/pkg/apis/meta"
33+
"github.com/fluxcd/pkg/runtime/metrics"
3334

3435
"github.com/fluxcd/notification-controller/api/v1beta1"
3536
)
3637

3738
// ReceiverReconciler reconciles a Receiver object
3839
type ReceiverReconciler struct {
3940
client.Client
40-
Log logr.Logger
41-
Scheme *runtime.Scheme
41+
Log logr.Logger
42+
Scheme *runtime.Scheme
43+
MetricsRecorder *metrics.Recorder
4244
}
4345

4446
// +kubebuilder:rbac:groups=notification.toolkit.fluxcd.io,resources=receivers,verbs=get;list;watch;create;update;patch;delete

‎go.mod‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,11 +8,11 @@ require (
88
github.com/fluxcd/notification-controller/api v0.1.0
99
github.com/fluxcd/pkg/apis/meta v0.0.2
1010
github.com/fluxcd/pkg/recorder v0.0.6
11-
github.com/fluxcd/pkg/runtime v0.0.6
11+
github.com/fluxcd/pkg/runtime v0.1.0
1212
github.com/fluxcd/source-controller/api v0.1.0
1313
github.com/go-logr/logr v0.1.0
1414
github.com/google/go-github/v32 v32.0.0
15-
github.com/hashicorp/go-retryablehttp v0.6.6
15+
github.com/hashicorp/go-retryablehttp v0.6.7
1616
github.com/onsi/ginkgo v1.12.1
1717
github.com/onsi/gomega v1.10.1
1818
github.com/stretchr/testify v1.6.1

‎go.sum‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -70,8 +70,8 @@ github.com/fluxcd/pkg/apis/meta v0.0.2 h1:kyA4Y0IzNjf1joBOnFqpWG7aNDHvtLExZcaHQM
7070
github.com/fluxcd/pkg/apis/meta v0.0.2/go.mod h1:nCNps5JJOcEQr3MNDmZqI4o0chjePSUYL6Q2ktDtotU=
7171
github.com/fluxcd/pkg/recorder v0.0.6 h1:me/n8syeeGXz50OXoPX3jgIj9AtinvhHdKT9Dy+MbHs=
7272
github.com/fluxcd/pkg/recorder v0.0.6/go.mod h1:IfQxfVRSNsWs3B0Yp5B6ObEWwKHILlAx8N7XkoDdhFg=
73-
github.com/fluxcd/pkg/runtime v0.0.6 h1:m7qwr2wRePs1vzVlM0Y88vitXSsv1lb3QCJflRpa3qQ=
74-
github.com/fluxcd/pkg/runtime v0.0.6/go.mod h1:iLjncjktQVpqpb1NsY2fW+UYDFOtVyt+yJrxqrrK8A0=
73+
github.com/fluxcd/pkg/runtime v0.1.0 h1:mCLj5GlQZqWtK3tvtZTmfgFOLsTUY1iqg3CmEyS1nRs=
74+
github.com/fluxcd/pkg/runtime v0.1.0/go.mod h1:OXkrYtDLw3GhclbzvnzfSktUyxRmC3FFhXj0tVVaIX8=
7575
github.com/fluxcd/source-controller/api v0.1.0 h1:ky3gMs3mnkDl6ClX+7uT2BNxU+sLzW/6a8B/M1KfySw=
7676
github.com/fluxcd/source-controller/api v0.1.0/go.mod h1:1ac/vj49YVPKF+xBHTo/9pfFj64TcLc1RLaxi4MwVEM=
7777
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
@@ -190,6 +190,8 @@ github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrj
190190
github.com/hashicorp/go-retryablehttp v0.6.4/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY=
191191
github.com/hashicorp/go-retryablehttp v0.6.6 h1:HJunrbHTDDbBb/ay4kxa1n+dLmttUlnP3V9oNE4hmsM=
192192
github.com/hashicorp/go-retryablehttp v0.6.6/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY=
193+
github.com/hashicorp/go-retryablehttp v0.6.7 h1:8/CAEZt/+F7kR7GevNHulKkUjLht3CPmn7egmhieNKo=
194+
github.com/hashicorp/go-retryablehttp v0.6.7/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY=
193195
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
194196
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
195197
github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=

‎main.go‎

Lines changed: 20 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -24,12 +24,15 @@ import (
2424
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
2525
_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
2626
ctrl "sigs.k8s.io/controller-runtime"
27+
crtlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
28+
29+
"github.com/fluxcd/pkg/runtime/logger"
30+
"github.com/fluxcd/pkg/runtime/metrics"
31+
sourcev1 "github.com/fluxcd/source-controller/api/v1beta1"
2732

2833
"github.com/fluxcd/notification-controller/api/v1beta1"
2934
"github.com/fluxcd/notification-controller/controllers"
3035
"github.com/fluxcd/notification-controller/internal/server"
31-
"github.com/fluxcd/pkg/runtime/logger"
32-
sourcev1 "github.com/fluxcd/source-controller/api/v1beta1"
3336
// +kubebuilder:scaffold:imports
3437
)
3538

@@ -74,6 +77,9 @@ func main() {
7477
zapLogger := logger.NewLogger(logLevel, logJSON)
7578
ctrl.SetLogger(zapLogger)
7679

80+
metricsRecorder := metrics.NewRecorder()
81+
crtlmetrics.Registry.MustRegister(metricsRecorder.Collectors()...)
82+
7783
watchNamespace := ""
7884
if !watchAllNamespaces {
7985
watchNamespace = os.Getenv("RUNTIME_NAMESPACE")
@@ -94,25 +100,28 @@ func main() {
94100
}
95101

96102
if err = (&controllers.ProviderReconciler{
97-
Client: mgr.GetClient(),
98-
Log: ctrl.Log.WithName("controllers").WithName("Provider"),
99-
Scheme: mgr.GetScheme(),
103+
Client: mgr.GetClient(),
104+
Log: ctrl.Log.WithName("controllers").WithName("Provider"),
105+
Scheme: mgr.GetScheme(),
106+
MetricsRecorder: metricsRecorder,
100107
}).SetupWithManager(mgr); err != nil {
101108
setupLog.Error(err, "unable to create controller", "controller", "Provider")
102109
os.Exit(1)
103110
}
104111
if err = (&controllers.AlertReconciler{
105-
Client: mgr.GetClient(),
106-
Log: ctrl.Log.WithName("controllers").WithName("Alert"),
107-
Scheme: mgr.GetScheme(),
112+
Client: mgr.GetClient(),
113+
Log: ctrl.Log.WithName("controllers").WithName("Alert"),
114+
Scheme: mgr.GetScheme(),
115+
MetricsRecorder: metricsRecorder,
108116
}).SetupWithManager(mgr); err != nil {
109117
setupLog.Error(err, "unable to create controller", "controller", "Alert")
110118
os.Exit(1)
111119
}
112120
if err = (&controllers.ReceiverReconciler{
113-
Client: mgr.GetClient(),
114-
Log: ctrl.Log.WithName("controllers").WithName("Receiver"),
115-
Scheme: mgr.GetScheme(),
121+
Client: mgr.GetClient(),
122+
Log: ctrl.Log.WithName("controllers").WithName("Receiver"),
123+
Scheme: mgr.GetScheme(),
124+
MetricsRecorder: metricsRecorder,
116125
}).SetupWithManager(mgr); err != nil {
117126
setupLog.Error(err, "unable to create controller", "controller", "Receiver")
118127
os.Exit(1)

0 commit comments

Comments
 (0)