Skip to content

Commit f0ded74

Browse files
britaniarbritaniar
committed
add placement status metrics
1 parent 33f6aa1 commit f0ded74

File tree

3 files changed

+54
-8
lines changed

3 files changed

+54
-8
lines changed

‎pkg/controllers/clusterresourceplacement/controller.go‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -224,7 +224,7 @@ func (r *Reconciler) handleUpdate(ctx context.Context, crp *fleetv1beta1.Cluster
224224
klog.V(2).InfoS("Placement has finished the rollout process and reached the desired status", "clusterResourcePlacement", crpKObj, "generation", crp.Generation)
225225
r.Recorder.Event(crp, corev1.EventTypeNormal, "PlacementRolloutCompleted", "Placement has finished the rollout process and reached the desired status")
226226
}
227-
metrics.FleetPlacementComplete.WithLabelValues(crp.Name).Set(1)
227+
metrics.FleetPlacementComplete.WithLabelValues(crp.Name).SetToCurrentTime()
228228
// We don't need to requeue any request now by watching the binding changes
229229
return ctrl.Result{}, nil
230230
}
@@ -1029,8 +1029,7 @@ func isRolloutCompleted(crp *fleetv1beta1.ClusterResourcePlacement) bool {
10291029
for _, i := range expectedCondTypes {
10301030
cond := crp.GetCondition(string(i.ClusterResourcePlacementConditionType()))
10311031
if cond != nil {
1032-
metrics.FleetPlacementStatus.DeletePartialMatch(prometheus.Labels{"name": crp.Name, "conditionType": cond.Type})
1033-
metrics.FleetPlacementStatus.WithLabelValues(crp.Name, strconv.FormatInt(crp.Generation, 10), cond.Type, string(cond.Status), strconv.FormatInt(cond.ObservedGeneration, 10)).Set(float64(cond.LastTransitionTime.UnixNano()) / 1e9)
1032+
emitPlacementStatusMetric(cond, crp)
10341033
}
10351034
if !condition.IsConditionStatusTrue(cond, crp.Generation) {
10361035
return false
@@ -1042,8 +1041,12 @@ func isRolloutCompleted(crp *fleetv1beta1.ClusterResourcePlacement) bool {
10421041
func isCRPScheduled(crp *fleetv1beta1.ClusterResourcePlacement) bool {
10431042
cond := crp.GetCondition(string(fleetv1beta1.ClusterResourcePlacementScheduledConditionType))
10441043
if cond != nil {
1045-
metrics.FleetPlacementStatus.DeletePartialMatch(prometheus.Labels{"name": crp.Name, "conditionType": cond.Type})
1046-
metrics.FleetPlacementStatus.WithLabelValues(crp.Name, strconv.FormatInt(crp.Generation, 10), cond.Type, string(cond.Status), strconv.FormatInt(cond.ObservedGeneration, 10)).Set(float64(cond.LastTransitionTime.UnixNano()) / 1e9)
1044+
emitPlacementStatusMetric(cond, crp)
10471045
}
10481046
return condition.IsConditionStatusTrue(cond, crp.Generation)
10491047
}
1048+
1049+
func emitPlacementStatusMetric(cond *metav1.Condition, crp *fleetv1beta1.ClusterResourcePlacement) {
1050+
metrics.FleetPlacementStatus.DeletePartialMatch(prometheus.Labels{"name": crp.Name, "conditionType": cond.Type})
1051+
metrics.FleetPlacementStatus.WithLabelValues(crp.Name, strconv.FormatInt(crp.Generation, 10), cond.Type, strconv.FormatInt(cond.ObservedGeneration, 10), string(cond.Status), cond.Reason).Set(float64(cond.LastTransitionTime.UnixNano() / 1e9))
1052+
}

‎pkg/controllers/clusterresourceplacement/controller_test.go‎

Lines changed: 45 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,13 @@ import (
1111
"errors"
1212
"fmt"
1313
"strconv"
14+
"strings"
1415
"testing"
16+
"time"
1517

1618
"github.com/google/go-cmp/cmp"
1719
"github.com/google/go-cmp/cmp/cmpopts"
20+
"github.com/prometheus/client_golang/prometheus/testutil"
1821
corev1 "k8s.io/api/core/v1"
1922
apierrors "k8s.io/apimachinery/pkg/api/errors"
2023
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -28,6 +31,7 @@ import (
2831

2932
fleetv1beta1 "go.goms.io/fleet/apis/placement/v1beta1"
3033
"go.goms.io/fleet/pkg/utils/controller"
34+
"go.goms.io/fleet/pkg/utils/controller/metrics"
3135
"go.goms.io/fleet/pkg/utils/defaulter"
3236
"go.goms.io/fleet/test/utils/resource"
3337
)
@@ -3073,7 +3077,9 @@ func TestHandleDelete(t *testing.T) {
30733077
}
30743078

30753079
func TestIsRolloutComplete(t *testing.T) {
3080+
metricMetadata := "\n# HELP fleet_workload_placement_status Placement status\n# TYPE fleet_workload_placement_status gauge"
30763081
crpGeneration := int64(25)
3082+
currentTime := metav1.Now()
30773083
tests := []struct {
30783084
name string
30793085
conditions []metav1.Condition
@@ -3086,31 +3092,37 @@ func TestIsRolloutComplete(t *testing.T) {
30863092
Status: metav1.ConditionTrue,
30873093
Type: string(fleetv1beta1.ClusterResourcePlacementAppliedConditionType),
30883094
ObservedGeneration: crpGeneration,
3095+
LastTransitionTime: metav1.NewTime(currentTime.Add(time.Second * 5)),
30893096
},
30903097
{
30913098
Status: metav1.ConditionTrue,
30923099
Type: string(fleetv1beta1.ClusterResourcePlacementAvailableConditionType),
30933100
ObservedGeneration: crpGeneration,
3101+
LastTransitionTime: metav1.NewTime(currentTime.Add(time.Second * 4)),
30943102
},
30953103
{
30963104
Status: metav1.ConditionTrue,
30973105
Type: string(fleetv1beta1.ClusterResourcePlacementOverriddenConditionType),
30983106
ObservedGeneration: crpGeneration,
3107+
LastTransitionTime: metav1.NewTime(currentTime.Add(time.Second * 2)),
30993108
},
31003109
{
31013110
Status: metav1.ConditionTrue,
31023111
Type: string(fleetv1beta1.ClusterResourcePlacementRolloutStartedConditionType),
31033112
ObservedGeneration: crpGeneration,
3113+
LastTransitionTime: metav1.NewTime(currentTime.Add(time.Second)),
31043114
},
31053115
{
31063116
Status: metav1.ConditionTrue,
31073117
Type: string(fleetv1beta1.ClusterResourcePlacementScheduledConditionType),
31083118
ObservedGeneration: crpGeneration,
3119+
LastTransitionTime: currentTime,
31093120
},
31103121
{
31113122
Status: metav1.ConditionTrue,
31123123
Type: string(fleetv1beta1.ClusterResourcePlacementWorkSynchronizedConditionType),
31133124
ObservedGeneration: crpGeneration,
3125+
LastTransitionTime: metav1.NewTime(currentTime.Add(time.Second * 3)),
31143126
},
31153127
},
31163128
want: true,
@@ -3122,6 +3134,7 @@ func TestIsRolloutComplete(t *testing.T) {
31223134
Status: metav1.ConditionUnknown,
31233135
Type: string(fleetv1beta1.ClusterResourcePlacementScheduledConditionType),
31243136
ObservedGeneration: crpGeneration,
3137+
LastTransitionTime: currentTime,
31253138
},
31263139
},
31273140
want: false,
@@ -3133,6 +3146,7 @@ func TestIsRolloutComplete(t *testing.T) {
31333146
Status: metav1.ConditionTrue,
31343147
Type: string(fleetv1beta1.ClusterResourcePlacementScheduledConditionType),
31353148
ObservedGeneration: crpGeneration,
3149+
LastTransitionTime: currentTime,
31363150
},
31373151
},
31383152
want: false,
@@ -3144,16 +3158,19 @@ func TestIsRolloutComplete(t *testing.T) {
31443158
Status: metav1.ConditionTrue,
31453159
Type: string(fleetv1beta1.ClusterResourcePlacementOverriddenConditionType),
31463160
ObservedGeneration: 1,
3161+
LastTransitionTime: metav1.NewTime(currentTime.Add(time.Second * 2)),
31473162
},
31483163
{
31493164
Status: metav1.ConditionTrue,
31503165
Type: string(fleetv1beta1.ClusterResourcePlacementRolloutStartedConditionType),
31513166
ObservedGeneration: crpGeneration,
3167+
LastTransitionTime: metav1.NewTime(currentTime.Add(time.Second)),
31523168
},
31533169
{
31543170
Status: metav1.ConditionTrue,
31553171
Type: string(fleetv1beta1.ClusterResourcePlacementScheduledConditionType),
31563172
ObservedGeneration: crpGeneration,
3173+
LastTransitionTime: currentTime,
31573174
},
31583175
},
31593176
want: false,
@@ -3165,21 +3182,25 @@ func TestIsRolloutComplete(t *testing.T) {
31653182
Status: metav1.ConditionTrue,
31663183
Type: string(fleetv1beta1.ClusterResourcePlacementOverriddenConditionType),
31673184
ObservedGeneration: crpGeneration,
3185+
LastTransitionTime: metav1.NewTime(currentTime.Add(time.Second * 2)),
31683186
},
31693187
{
31703188
Status: metav1.ConditionTrue,
31713189
Type: string(fleetv1beta1.ClusterResourcePlacementRolloutStartedConditionType),
31723190
ObservedGeneration: crpGeneration,
3191+
LastTransitionTime: metav1.NewTime(currentTime.Add(time.Second)),
31733192
},
31743193
{
31753194
Status: metav1.ConditionTrue,
31763195
Type: string(fleetv1beta1.ClusterResourcePlacementScheduledConditionType),
31773196
ObservedGeneration: crpGeneration,
3197+
LastTransitionTime: currentTime,
31783198
},
31793199
{
31803200
Status: metav1.ConditionFalse,
31813201
Type: string(fleetv1beta1.ClusterResourcePlacementWorkSynchronizedConditionType),
31823202
ObservedGeneration: crpGeneration,
3203+
LastTransitionTime: metav1.NewTime(currentTime.Add(time.Second * 3)),
31833204
},
31843205
},
31853206
want: false,
@@ -3191,21 +3212,25 @@ func TestIsRolloutComplete(t *testing.T) {
31913212
Status: metav1.ConditionTrue,
31923213
Type: string(fleetv1beta1.ClusterResourcePlacementOverriddenConditionType),
31933214
ObservedGeneration: crpGeneration,
3215+
LastTransitionTime: metav1.NewTime(currentTime.Add(time.Second * 2)),
31943216
},
31953217
{
31963218
Status: metav1.ConditionTrue,
31973219
Type: string(fleetv1beta1.ClusterResourcePlacementRolloutStartedConditionType),
31983220
ObservedGeneration: crpGeneration,
3221+
LastTransitionTime: metav1.NewTime(currentTime.Add(time.Second)),
31993222
},
32003223
{
32013224
Status: metav1.ConditionTrue,
32023225
Type: string(fleetv1beta1.ClusterResourcePlacementScheduledConditionType),
32033226
ObservedGeneration: crpGeneration,
3227+
LastTransitionTime: currentTime,
32043228
},
32053229
{
32063230
Status: metav1.ConditionTrue,
32073231
Type: string(fleetv1beta1.ClusterResourcePlacementWorkSynchronizedConditionType),
32083232
ObservedGeneration: crpGeneration,
3233+
LastTransitionTime: metav1.NewTime(currentTime.Add(time.Second * 3)),
32093234
},
32103235
},
32113236
want: false,
@@ -3217,41 +3242,47 @@ func TestIsRolloutComplete(t *testing.T) {
32173242
Status: metav1.ConditionTrue,
32183243
Type: string(fleetv1beta1.ClusterResourcePlacementAppliedConditionType),
32193244
ObservedGeneration: crpGeneration,
3245+
LastTransitionTime: metav1.NewTime(currentTime.Add(time.Second * 5)),
32203246
},
32213247
{
32223248
Status: metav1.ConditionFalse,
32233249
Type: string(fleetv1beta1.ClusterResourcePlacementAvailableConditionType),
32243250
ObservedGeneration: crpGeneration,
3251+
LastTransitionTime: metav1.NewTime(currentTime.Add(time.Second * 4)),
32253252
},
32263253
{
32273254
Status: metav1.ConditionTrue,
32283255
Type: string(fleetv1beta1.ClusterResourcePlacementOverriddenConditionType),
32293256
ObservedGeneration: crpGeneration,
3257+
LastTransitionTime: metav1.NewTime(currentTime.Add(time.Second * 2)),
32303258
},
32313259
{
32323260
Status: metav1.ConditionTrue,
32333261
Type: string(fleetv1beta1.ClusterResourcePlacementRolloutStartedConditionType),
32343262
ObservedGeneration: crpGeneration,
3263+
LastTransitionTime: metav1.NewTime(currentTime.Add(time.Second)),
32353264
},
32363265
{
32373266
Status: metav1.ConditionTrue,
32383267
Type: string(fleetv1beta1.ClusterResourcePlacementScheduledConditionType),
32393268
ObservedGeneration: crpGeneration,
3269+
LastTransitionTime: currentTime,
32403270
},
32413271
{
32423272
Status: metav1.ConditionTrue,
32433273
Type: string(fleetv1beta1.ClusterResourcePlacementWorkSynchronizedConditionType),
32443274
ObservedGeneration: crpGeneration,
3275+
LastTransitionTime: metav1.NewTime(currentTime.Add(time.Second * 3)),
32453276
},
32463277
},
32473278
want: false,
32483279
},
32493280
}
3250-
for _, tc := range tests {
3281+
for i, tc := range tests {
32513282
t.Run(tc.name, func(t *testing.T) {
32523283
crp := &fleetv1beta1.ClusterResourcePlacement{
32533284
ObjectMeta: metav1.ObjectMeta{
3254-
Name: testName,
3285+
Name: testName + strconv.Itoa(i),
32553286
Generation: crpGeneration,
32563287
},
32573288
Status: fleetv1beta1.ClusterResourcePlacementStatus{
@@ -3262,6 +3293,18 @@ func TestIsRolloutComplete(t *testing.T) {
32623293
if got != tc.want {
32633294
t.Errorf("isRolloutCompleted() got %v, want %v", got, tc.want)
32643295
}
3296+
3297+
var wantMetrics string
3298+
for _, cond := range tc.conditions {
3299+
lastTransitionTime := float64(cond.LastTransitionTime.UnixNano() / 1e9)
3300+
wantMetrics += fmt.Sprintf("\nfleet_workload_placement_status{conditionType=\"%[1]s\",generation=\"%[2]d\",name=\"%[3]s\",observedGeneration=\"%[4]d\",reason=\"%[5]s\",status=\"%[6]s\"} %f",
3301+
cond.Type, crp.Generation, crp.Name, cond.ObservedGeneration, cond.Reason, cond.Status, lastTransitionTime)
3302+
}
3303+
3304+
if err := testutil.CollectAndCompare(metrics.FleetPlacementStatus, strings.NewReader(metricMetadata+wantMetrics+"\n")); err != nil {
3305+
t.Errorf("%s", err)
3306+
}
3307+
metrics.FleetPlacementStatus.Reset()
32653308
})
32663309
}
32673310
}

‎pkg/utils/controller/metrics/metrics.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ var (
6060
FleetPlacementStatus = prometheus.NewGaugeVec(prometheus.GaugeOpts{
6161
Name: "fleet_workload_placement_status",
6262
Help: "Placement status",
63-
}, []string{"name", "generation", "conditionType", "status", "observedGeneration"})
63+
}, []string{"name", "generation", "conditionType", "observedGeneration", "status", "reason"})
6464
)
6565

6666
func init() {

0 commit comments

Comments
 (0)