Skip to content

Commit 42093bd

Browse files
britaniarbritaniar
committed
add placement status metrics
1 parent 6160725 commit 42093bd

File tree

3 files changed

+73
-8
lines changed

3 files changed

+73
-8
lines changed

‎pkg/controllers/clusterresourceplacement/controller.go‎

Lines changed: 19 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,8 @@ func (r *Reconciler) handleDelete(ctx context.Context, crp *fleetv1beta1.Cluster
101101
}
102102
klog.V(2).InfoS("Removed crp-cleanup finalizer", "clusterResourcePlacement", crpKObj)
103103
r.Recorder.Event(crp, corev1.EventTypeNormal, "PlacementCleanupFinalizerRemoved", "Deleted the snapshots and removed the placement cleanup finalizer")
104-
metrics.FleetPlacementStatus.Delete(prometheus.Labels{"name": crp.Name})
104+
metrics.FleetPlacementComplete.Delete(prometheus.Labels{"name": crp.Name})
105+
metrics.FleetPlacementStatus.DeletePartialMatch(prometheus.Labels{"name": crp.Name})
105106
return ctrl.Result{}, nil
106107
}
107108

@@ -222,12 +223,12 @@ func (r *Reconciler) handleUpdate(ctx context.Context, crp *fleetv1beta1.Cluster
222223
klog.V(2).InfoS("Placement has finished the rollout process and reached the desired status", "clusterResourcePlacement", crpKObj, "generation", crp.Generation)
223224
r.Recorder.Event(crp, corev1.EventTypeNormal, "PlacementRolloutCompleted", "Placement has finished the rollout process and reached the desired status")
224225
}
225-
metrics.FleetPlacementStatus.WithLabelValues(crp.Name).Set(1)
226+
metrics.FleetPlacementComplete.WithLabelValues(crp.Name).SetToCurrentTime()
226227
// We don't need to requeue any request now by watching the binding changes
227228
return ctrl.Result{}, nil
228229
}
229230

230-
metrics.FleetPlacementStatus.WithLabelValues(crp.Name).Set(0)
231+
metrics.FleetPlacementComplete.WithLabelValues(crp.Name).Set(0)
231232
if !isClusterScheduled {
232233
// Note:
233234
// If the scheduledCondition is failed, it means the placement requirement cannot be satisfied fully. For example,
@@ -1025,13 +1026,26 @@ func isRolloutCompleted(crp *fleetv1beta1.ClusterResourcePlacement) bool {
10251026

10261027
expectedCondTypes := determineExpectedCRPAndResourcePlacementStatusCondType(crp)
10271028
for _, i := range expectedCondTypes {
1028-
if !condition.IsConditionStatusTrue(crp.GetCondition(string(i.ClusterResourcePlacementConditionType())), crp.Generation) {
1029+
cond := crp.GetCondition(string(i.ClusterResourcePlacementConditionType()))
1030+
if cond != nil {
1031+
emitPlacementStatusMetric(cond, crp)
1032+
}
1033+
if !condition.IsConditionStatusTrue(cond, crp.Generation) {
10291034
return false
10301035
}
10311036
}
10321037
return true
10331038
}
10341039

10351040
func isCRPScheduled(crp *fleetv1beta1.ClusterResourcePlacement) bool {
1036-
return condition.IsConditionStatusTrue(crp.GetCondition(string(fleetv1beta1.ClusterResourcePlacementScheduledConditionType)), crp.Generation)
1041+
cond := crp.GetCondition(string(fleetv1beta1.ClusterResourcePlacementScheduledConditionType))
1042+
if cond != nil {
1043+
emitPlacementStatusMetric(cond, crp)
1044+
}
1045+
return condition.IsConditionStatusTrue(cond, crp.Generation)
1046+
}
1047+
1048+
func emitPlacementStatusMetric(cond *metav1.Condition, crp *fleetv1beta1.ClusterResourcePlacement) {
1049+
metrics.FleetPlacementStatus.DeletePartialMatch(prometheus.Labels{"name": crp.Name, "conditionType": cond.Type})
1050+
metrics.FleetPlacementStatus.WithLabelValues(crp.Name, strconv.FormatInt(crp.Generation, 10), cond.Type, strconv.FormatInt(cond.ObservedGeneration, 10), string(cond.Status), cond.Reason).Set(float64(cond.LastTransitionTime.UnixNano() / 1e9))
10371051
}

‎pkg/controllers/clusterresourceplacement/controller_test.go‎

Lines changed: 45 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,13 @@ import (
1111
"errors"
1212
"fmt"
1313
"strconv"
14+
"strings"
1415
"testing"
16+
"time"
1517

1618
"github.com/google/go-cmp/cmp"
1719
"github.com/google/go-cmp/cmp/cmpopts"
20+
"github.com/prometheus/client_golang/prometheus/testutil"
1821
corev1 "k8s.io/api/core/v1"
1922
apierrors "k8s.io/apimachinery/pkg/api/errors"
2023
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -28,6 +31,7 @@ import (
2831

2932
fleetv1beta1 "go.goms.io/fleet/apis/placement/v1beta1"
3033
"go.goms.io/fleet/pkg/utils/controller"
34+
"go.goms.io/fleet/pkg/utils/controller/metrics"
3135
"go.goms.io/fleet/pkg/utils/defaulter"
3236
"go.goms.io/fleet/test/utils/resource"
3337
)
@@ -3073,7 +3077,9 @@ func TestHandleDelete(t *testing.T) {
30733077
}
30743078

30753079
func TestIsRolloutComplete(t *testing.T) {
3080+
metricMetadata := "\n# HELP fleet_workload_placement_status Placement status\n# TYPE fleet_workload_placement_status gauge"
30763081
crpGeneration := int64(25)
3082+
currentTime := metav1.Now()
30773083
tests := []struct {
30783084
name string
30793085
conditions []metav1.Condition
@@ -3086,31 +3092,37 @@ func TestIsRolloutComplete(t *testing.T) {
30863092
Status: metav1.ConditionTrue,
30873093
Type: string(fleetv1beta1.ClusterResourcePlacementAppliedConditionType),
30883094
ObservedGeneration: crpGeneration,
3095+
LastTransitionTime: metav1.NewTime(currentTime.Add(time.Second * 5)),
30893096
},
30903097
{
30913098
Status: metav1.ConditionTrue,
30923099
Type: string(fleetv1beta1.ClusterResourcePlacementAvailableConditionType),
30933100
ObservedGeneration: crpGeneration,
3101+
LastTransitionTime: metav1.NewTime(currentTime.Add(time.Second * 4)),
30943102
},
30953103
{
30963104
Status: metav1.ConditionTrue,
30973105
Type: string(fleetv1beta1.ClusterResourcePlacementOverriddenConditionType),
30983106
ObservedGeneration: crpGeneration,
3107+
LastTransitionTime: metav1.NewTime(currentTime.Add(time.Second * 2)),
30993108
},
31003109
{
31013110
Status: metav1.ConditionTrue,
31023111
Type: string(fleetv1beta1.ClusterResourcePlacementRolloutStartedConditionType),
31033112
ObservedGeneration: crpGeneration,
3113+
LastTransitionTime: metav1.NewTime(currentTime.Add(time.Second)),
31043114
},
31053115
{
31063116
Status: metav1.ConditionTrue,
31073117
Type: string(fleetv1beta1.ClusterResourcePlacementScheduledConditionType),
31083118
ObservedGeneration: crpGeneration,
3119+
LastTransitionTime: currentTime,
31093120
},
31103121
{
31113122
Status: metav1.ConditionTrue,
31123123
Type: string(fleetv1beta1.ClusterResourcePlacementWorkSynchronizedConditionType),
31133124
ObservedGeneration: crpGeneration,
3125+
LastTransitionTime: metav1.NewTime(currentTime.Add(time.Second * 3)),
31143126
},
31153127
},
31163128
want: true,
@@ -3122,6 +3134,7 @@ func TestIsRolloutComplete(t *testing.T) {
31223134
Status: metav1.ConditionUnknown,
31233135
Type: string(fleetv1beta1.ClusterResourcePlacementScheduledConditionType),
31243136
ObservedGeneration: crpGeneration,
3137+
LastTransitionTime: currentTime,
31253138
},
31263139
},
31273140
want: false,
@@ -3133,6 +3146,7 @@ func TestIsRolloutComplete(t *testing.T) {
31333146
Status: metav1.ConditionTrue,
31343147
Type: string(fleetv1beta1.ClusterResourcePlacementScheduledConditionType),
31353148
ObservedGeneration: crpGeneration,
3149+
LastTransitionTime: currentTime,
31363150
},
31373151
},
31383152
want: false,
@@ -3144,16 +3158,19 @@ func TestIsRolloutComplete(t *testing.T) {
31443158
Status: metav1.ConditionTrue,
31453159
Type: string(fleetv1beta1.ClusterResourcePlacementOverriddenConditionType),
31463160
ObservedGeneration: 1,
3161+
LastTransitionTime: metav1.NewTime(currentTime.Add(time.Second * 2)),
31473162
},
31483163
{
31493164
Status: metav1.ConditionTrue,
31503165
Type: string(fleetv1beta1.ClusterResourcePlacementRolloutStartedConditionType),
31513166
ObservedGeneration: crpGeneration,
3167+
LastTransitionTime: metav1.NewTime(currentTime.Add(time.Second)),
31523168
},
31533169
{
31543170
Status: metav1.ConditionTrue,
31553171
Type: string(fleetv1beta1.ClusterResourcePlacementScheduledConditionType),
31563172
ObservedGeneration: crpGeneration,
3173+
LastTransitionTime: currentTime,
31573174
},
31583175
},
31593176
want: false,
@@ -3165,21 +3182,25 @@ func TestIsRolloutComplete(t *testing.T) {
31653182
Status: metav1.ConditionTrue,
31663183
Type: string(fleetv1beta1.ClusterResourcePlacementOverriddenConditionType),
31673184
ObservedGeneration: crpGeneration,
3185+
LastTransitionTime: metav1.NewTime(currentTime.Add(time.Second * 2)),
31683186
},
31693187
{
31703188
Status: metav1.ConditionTrue,
31713189
Type: string(fleetv1beta1.ClusterResourcePlacementRolloutStartedConditionType),
31723190
ObservedGeneration: crpGeneration,
3191+
LastTransitionTime: metav1.NewTime(currentTime.Add(time.Second)),
31733192
},
31743193
{
31753194
Status: metav1.ConditionTrue,
31763195
Type: string(fleetv1beta1.ClusterResourcePlacementScheduledConditionType),
31773196
ObservedGeneration: crpGeneration,
3197+
LastTransitionTime: currentTime,
31783198
},
31793199
{
31803200
Status: metav1.ConditionFalse,
31813201
Type: string(fleetv1beta1.ClusterResourcePlacementWorkSynchronizedConditionType),
31823202
ObservedGeneration: crpGeneration,
3203+
LastTransitionTime: metav1.NewTime(currentTime.Add(time.Second * 3)),
31833204
},
31843205
},
31853206
want: false,
@@ -3191,21 +3212,25 @@ func TestIsRolloutComplete(t *testing.T) {
31913212
Status: metav1.ConditionTrue,
31923213
Type: string(fleetv1beta1.ClusterResourcePlacementOverriddenConditionType),
31933214
ObservedGeneration: crpGeneration,
3215+
LastTransitionTime: metav1.NewTime(currentTime.Add(time.Second * 2)),
31943216
},
31953217
{
31963218
Status: metav1.ConditionTrue,
31973219
Type: string(fleetv1beta1.ClusterResourcePlacementRolloutStartedConditionType),
31983220
ObservedGeneration: crpGeneration,
3221+
LastTransitionTime: metav1.NewTime(currentTime.Add(time.Second)),
31993222
},
32003223
{
32013224
Status: metav1.ConditionTrue,
32023225
Type: string(fleetv1beta1.ClusterResourcePlacementScheduledConditionType),
32033226
ObservedGeneration: crpGeneration,
3227+
LastTransitionTime: currentTime,
32043228
},
32053229
{
32063230
Status: metav1.ConditionTrue,
32073231
Type: string(fleetv1beta1.ClusterResourcePlacementWorkSynchronizedConditionType),
32083232
ObservedGeneration: crpGeneration,
3233+
LastTransitionTime: metav1.NewTime(currentTime.Add(time.Second * 3)),
32093234
},
32103235
},
32113236
want: false,
@@ -3217,41 +3242,47 @@ func TestIsRolloutComplete(t *testing.T) {
32173242
Status: metav1.ConditionTrue,
32183243
Type: string(fleetv1beta1.ClusterResourcePlacementAppliedConditionType),
32193244
ObservedGeneration: crpGeneration,
3245+
LastTransitionTime: metav1.NewTime(currentTime.Add(time.Second * 5)),
32203246
},
32213247
{
32223248
Status: metav1.ConditionFalse,
32233249
Type: string(fleetv1beta1.ClusterResourcePlacementAvailableConditionType),
32243250
ObservedGeneration: crpGeneration,
3251+
LastTransitionTime: metav1.NewTime(currentTime.Add(time.Second * 4)),
32253252
},
32263253
{
32273254
Status: metav1.ConditionTrue,
32283255
Type: string(fleetv1beta1.ClusterResourcePlacementOverriddenConditionType),
32293256
ObservedGeneration: crpGeneration,
3257+
LastTransitionTime: metav1.NewTime(currentTime.Add(time.Second * 2)),
32303258
},
32313259
{
32323260
Status: metav1.ConditionTrue,
32333261
Type: string(fleetv1beta1.ClusterResourcePlacementRolloutStartedConditionType),
32343262
ObservedGeneration: crpGeneration,
3263+
LastTransitionTime: metav1.NewTime(currentTime.Add(time.Second)),
32353264
},
32363265
{
32373266
Status: metav1.ConditionTrue,
32383267
Type: string(fleetv1beta1.ClusterResourcePlacementScheduledConditionType),
32393268
ObservedGeneration: crpGeneration,
3269+
LastTransitionTime: currentTime,
32403270
},
32413271
{
32423272
Status: metav1.ConditionTrue,
32433273
Type: string(fleetv1beta1.ClusterResourcePlacementWorkSynchronizedConditionType),
32443274
ObservedGeneration: crpGeneration,
3275+
LastTransitionTime: metav1.NewTime(currentTime.Add(time.Second * 3)),
32453276
},
32463277
},
32473278
want: false,
32483279
},
32493280
}
3250-
for _, tc := range tests {
3281+
for i, tc := range tests {
32513282
t.Run(tc.name, func(t *testing.T) {
32523283
crp := &fleetv1beta1.ClusterResourcePlacement{
32533284
ObjectMeta: metav1.ObjectMeta{
3254-
Name: testCRPName,
3285+
Name: testCRPName + strconv.Itoa(i),
32553286
Generation: crpGeneration,
32563287
},
32573288
Status: fleetv1beta1.ClusterResourcePlacementStatus{
@@ -3262,6 +3293,18 @@ func TestIsRolloutComplete(t *testing.T) {
32623293
if got != tc.want {
32633294
t.Errorf("isRolloutCompleted() got %v, want %v", got, tc.want)
32643295
}
3296+
3297+
var wantMetrics string
3298+
for _, cond := range tc.conditions {
3299+
lastTransitionTime := float64(cond.LastTransitionTime.UnixNano() / 1e9)
3300+
wantMetrics += fmt.Sprintf("\nfleet_workload_placement_status{conditionType=\"%[1]s\",generation=\"%[2]d\",name=\"%[3]s\",observedGeneration=\"%[4]d\",reason=\"%[5]s\",status=\"%[6]s\"} %f",
3301+
cond.Type, crp.Generation, crp.Name, cond.ObservedGeneration, cond.Reason, cond.Status, lastTransitionTime)
3302+
}
3303+
3304+
if err := testutil.CollectAndCompare(metrics.FleetPlacementStatus, strings.NewReader(metricMetadata+wantMetrics+"\n")); err != nil {
3305+
t.Errorf("%s", err)
3306+
}
3307+
metrics.FleetPlacementStatus.Reset()
32653308
})
32663309
}
32673310
}

‎pkg/utils/controller/metrics/metrics.go‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,11 +50,18 @@ var (
5050
Help: "Number of currently used workers per controller",
5151
}, []string{"controller"})
5252

53-
FleetPlacementStatus = prometheus.NewGaugeVec(prometheus.GaugeOpts{
53+
//FleetPlacementComplete is a prometheus metric which keeps track if the placement is complete
54+
FleetPlacementComplete = prometheus.NewGaugeVec(prometheus.GaugeOpts{
5455
Name: "fleet_workload_placement_complete",
5556
Help: "Placement complete status ",
5657
}, []string{"name"})
5758

59+
// FleetPlacementStatus is a prometheus metric which keeps track of the placement status
60+
FleetPlacementStatus = prometheus.NewGaugeVec(prometheus.GaugeOpts{
61+
Name: "fleet_workload_placement_status",
62+
Help: "Placement status",
63+
}, []string{"name", "generation", "conditionType", "observedGeneration", "status", "reason"})
64+
5865
// FleetEvictionStatus is prometheus metrics which holds the
5966
// status of eviction completion.
6067
FleetEvictionStatus = prometheus.NewGaugeVec(prometheus.GaugeOpts{
@@ -70,6 +77,7 @@ func init() {
7077
FleetReconcileTime,
7178
FleetWorkerCount,
7279
FleetActiveWorkers,
80+
FleetPlacementComplete,
7381
FleetPlacementStatus,
7482
FleetEvictionStatus,
7583
)

0 commit comments

Comments
 (0)