diff --git a/docs/metrics/workload/horizontalpodautoscaler-metrics.md b/docs/metrics/workload/horizontalpodautoscaler-metrics.md index 383bdca13f..c3fe059c0f 100644 --- a/docs/metrics/workload/horizontalpodautoscaler-metrics.md +++ b/docs/metrics/workload/horizontalpodautoscaler-metrics.md @@ -1,15 +1,15 @@ # Horizontal Pod Autoscaler Metrics | Metric name | Metric type | Description | Labels/tags | Status | -| ---------------------------------------------------- | ----------- | ------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------ | +| ---------------------------------------------------- | ----------- | ------------------------------------------------------------------------------------------------------------------------- |----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| ------------ | | kube_horizontalpodautoscaler_info | Gauge | | `horizontalpodautoscaler`=<hpa-name>
`namespace`=<hpa-namespace>
`scaletargetref_api_version`=<hpa-target-api-version>
`scaletargetref_kind`=<hpa-target-kind>
`scaletargetref_name`=<hpa-target-name> | EXPERIMENTAL | | kube_horizontalpodautoscaler_annotations | Gauge | Kubernetes annotations converted to Prometheus labels controlled via [--metric-annotations-allowlist](../../developer/cli-arguments.md) | `horizontalpodautoscaler`=<hpa-name>
`namespace`=<hpa-namespace> | EXPERIMENTAL | | kube_horizontalpodautoscaler_labels | Gauge | Kubernetes labels converted to Prometheus labels controlled via [--metric-labels-allowlist](../../developer/cli-arguments.md) | `horizontalpodautoscaler`=<hpa-name>
`namespace`=<hpa-namespace> | STABLE | | kube_horizontalpodautoscaler_metadata_generation | Gauge | | `horizontalpodautoscaler`=<hpa-name>
`namespace`=<hpa-namespace> | STABLE | | kube_horizontalpodautoscaler_spec_max_replicas | Gauge | | `horizontalpodautoscaler`=<hpa-name>
`namespace`=<hpa-namespace> | STABLE | | kube_horizontalpodautoscaler_spec_min_replicas | Gauge | | `horizontalpodautoscaler`=<hpa-name>
`namespace`=<hpa-namespace> | STABLE | -| kube_horizontalpodautoscaler_spec_target_metric | Gauge | | `horizontalpodautoscaler`=<hpa-name>
`namespace`=<hpa-namespace>
`metric_name`=<metric-name>
`metric_target_type`=<value\|utilization\|average> | EXPERIMENTAL | -| kube_horizontalpodautoscaler_status_target_metric | Gauge | | `horizontalpodautoscaler`=<hpa-name>
`namespace`=<hpa-namespace>
`metric_name`=<metric-name>
`metric_target_type`=<value\|utilization\|average> | EXPERIMENTAL | +| kube_horizontalpodautoscaler_spec_target_metric | Gauge | | `horizontalpodautoscaler`=<hpa-name>
`namespace`=<hpa-namespace>
`metric_name`=<metric-name>
`metric_target_type`=<value\|utilization\|average>
`selectorLabel_$labelKey`=<labelValue> | EXPERIMENTAL | +| kube_horizontalpodautoscaler_status_target_metric | Gauge | | `horizontalpodautoscaler`=<hpa-name>
`namespace`=<hpa-namespace>
`metric_name`=<metric-name>
`metric_target_type`=<value\|utilization\|average>
`selectorLabel_$labelKey`=<labelValue> | EXPERIMENTAL | | kube_horizontalpodautoscaler_status_condition | Gauge | | `horizontalpodautoscaler`=<hpa-name>
`namespace`=<hpa-namespace>
`condition`=<hpa-condition>
`status`=<true\|false\|unknown> | STABLE | | kube_horizontalpodautoscaler_status_current_replicas | Gauge | | `horizontalpodautoscaler`=<hpa-name>
`namespace`=<hpa-namespace> | STABLE | | kube_horizontalpodautoscaler_status_desired_replicas | Gauge | | `horizontalpodautoscaler`=<hpa-name>
`namespace`=<hpa-namespace> | STABLE | diff --git a/internal/store/horizontalpodautoscaler.go b/internal/store/horizontalpodautoscaler.go index 3994f14c39..7fc493fb10 100644 --- a/internal/store/horizontalpodautoscaler.go +++ b/internal/store/horizontalpodautoscaler.go @@ -194,14 +194,22 @@ func createHPASpecTargetMetric() generator.FamilyGenerator { var metricTarget autoscaling.MetricTarget // The variable maps the type of metric to the corresponding value metricMap := make(map[metricTargetType]float64) + // Object|Pods|External metrics can use the same name for mutltiple metrics. Add SelectorLabels to the metric labels + selectorLabels := make(map[string]string) switch m.Type { case autoscaling.ObjectMetricSourceType: metricName = m.Object.Metric.Name metricTarget = m.Object.Target + if m.Object.Metric.Selector != nil { + selectorLabels = m.Object.Metric.Selector.MatchLabels + } case autoscaling.PodsMetricSourceType: metricName = m.Pods.Metric.Name metricTarget = m.Pods.Target + if m.Pods.Metric.Selector != nil { + selectorLabels = m.Pods.Metric.Selector.MatchLabels + } case autoscaling.ResourceMetricSourceType: metricName = string(m.Resource.Name) metricTarget = m.Resource.Target @@ -211,6 +219,9 @@ func createHPASpecTargetMetric() generator.FamilyGenerator { case autoscaling.ExternalMetricSourceType: metricName = m.External.Metric.Name metricTarget = m.External.Target + if m.External.Metric.Selector != nil { + selectorLabels = m.External.Metric.Selector.MatchLabels + } default: // Skip unsupported metric type continue @@ -227,9 +238,10 @@ func createHPASpecTargetMetric() generator.FamilyGenerator { } for metricTypeIndex, metricValue := range metricMap { + selectorLabelKeys, selectorLabelValues := kubeMapToPrometheusLabels("selectorlabel", selectorLabels) ms = append(ms, &metric.Metric{ - LabelKeys: targetMetricLabels, - LabelValues: []string{metricName, metricTypeIndex.String()}, + LabelKeys: append(targetMetricLabels, selectorLabelKeys...), + LabelValues: append([]string{metricName, metricTypeIndex.String()}, selectorLabelValues...), Value: metricValue, }) } @@ -253,14 +265,22 @@ func createHPAStatusTargetMetric() generator.FamilyGenerator { var currentMetric autoscaling.MetricValueStatus // The variable maps the type of metric to the corresponding value metricMap := make(map[metricTargetType]float64) + // Object|Pods|External metrics can use the same name for mutltiple metrics. Add SelectorLabels to the metric labels + selectorLabels := make(map[string]string) switch m.Type { case autoscaling.ObjectMetricSourceType: metricName = m.Object.Metric.Name currentMetric = m.Object.Current + if m.Object.Metric.Selector != nil { + selectorLabels = m.Object.Metric.Selector.MatchLabels + } case autoscaling.PodsMetricSourceType: metricName = m.Pods.Metric.Name currentMetric = m.Pods.Current + if m.Pods.Metric.Selector != nil { + selectorLabels = m.Pods.Metric.Selector.MatchLabels + } case autoscaling.ResourceMetricSourceType: metricName = string(m.Resource.Name) currentMetric = m.Resource.Current @@ -270,6 +290,9 @@ func createHPAStatusTargetMetric() generator.FamilyGenerator { case autoscaling.ExternalMetricSourceType: metricName = m.External.Metric.Name currentMetric = m.External.Current + if m.External.Metric.Selector != nil { + selectorLabels = m.External.Metric.Selector.MatchLabels + } default: // Skip unsupported metric type continue @@ -286,9 +309,10 @@ func createHPAStatusTargetMetric() generator.FamilyGenerator { } for metricTypeIndex, metricValue := range metricMap { + selectorLabelKeys, selectorLabelValues := kubeMapToPrometheusLabels("selectorlabel", selectorLabels) ms = append(ms, &metric.Metric{ - LabelKeys: targetMetricLabels, - LabelValues: []string{metricName, metricTypeIndex.String()}, + LabelKeys: append(targetMetricLabels, selectorLabelKeys...), + LabelValues: append([]string{metricName, metricTypeIndex.String()}, selectorLabelValues...), Value: metricValue, }) } diff --git a/internal/store/horizontalpodautoscaler_test.go b/internal/store/horizontalpodautoscaler_test.go index 25903fe432..fa4713d0cf 100644 --- a/internal/store/horizontalpodautoscaler_test.go +++ b/internal/store/horizontalpodautoscaler_test.go @@ -79,6 +79,10 @@ func TestHPAStore(t *testing.T) { Object: &autoscaling.ObjectMetricSource{ Metric: autoscaling.MetricIdentifier{ Name: "hits", + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "foobar", + }}, }, Target: autoscaling.MetricTarget{ Value: resourcePtr(resource.MustParse("10")), @@ -103,6 +107,11 @@ func TestHPAStore(t *testing.T) { Pods: &autoscaling.PodsMetricSource{ Metric: autoscaling.MetricIdentifier{ Name: "transactions_processed", + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "foobar", + "k8s.io/name": "foobar", + }}, }, Target: autoscaling.MetricTarget{ AverageValue: resourcePtr(resource.MustParse("33")), @@ -150,6 +159,11 @@ func TestHPAStore(t *testing.T) { External: &autoscaling.ExternalMetricSource{ Metric: autoscaling.MetricIdentifier{ Name: "sqs_jobs", + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "foobar", + "job.1.type": "type1", + }}, }, Target: autoscaling.MetricTarget{ Value: resourcePtr(resource.MustParse("30")), @@ -215,15 +229,15 @@ func TestHPAStore(t *testing.T) { kube_horizontalpodautoscaler_spec_min_replicas{horizontalpodautoscaler="hpa1",namespace="ns1"} 2 kube_horizontalpodautoscaler_spec_target_metric{horizontalpodautoscaler="hpa1",metric_name="cpu",metric_target_type="utilization",namespace="ns1"} 80 kube_horizontalpodautoscaler_spec_target_metric{horizontalpodautoscaler="hpa1",metric_name="events",metric_target_type="average",namespace="ns1"} 30 - kube_horizontalpodautoscaler_spec_target_metric{horizontalpodautoscaler="hpa1",metric_name="hits",metric_target_type="average",namespace="ns1"} 12 - kube_horizontalpodautoscaler_spec_target_metric{horizontalpodautoscaler="hpa1",metric_name="hits",metric_target_type="value",namespace="ns1"} 10 + kube_horizontalpodautoscaler_spec_target_metric{horizontalpodautoscaler="hpa1",metric_name="hits",metric_target_type="average",namespace="ns1",selectorlabel_app="foobar"} 12 + kube_horizontalpodautoscaler_spec_target_metric{horizontalpodautoscaler="hpa1",metric_name="hits",metric_target_type="value",namespace="ns1",selectorlabel_app="foobar"} 10 kube_horizontalpodautoscaler_spec_target_metric{horizontalpodautoscaler="hpa1",metric_name="connections",metric_target_type="average",namespace="ns1"} 0.7 kube_horizontalpodautoscaler_spec_target_metric{horizontalpodautoscaler="hpa1",metric_name="connections",metric_target_type="value",namespace="ns1"} 0.5 kube_horizontalpodautoscaler_spec_target_metric{horizontalpodautoscaler="hpa1",metric_name="cpu",metric_target_type="utilization",namespace="ns1"} 80 kube_horizontalpodautoscaler_spec_target_metric{horizontalpodautoscaler="hpa1",metric_name="memory",metric_target_type="average",namespace="ns1"} 819200 kube_horizontalpodautoscaler_spec_target_metric{horizontalpodautoscaler="hpa1",metric_name="memory",metric_target_type="utilization",namespace="ns1"} 80 - kube_horizontalpodautoscaler_spec_target_metric{horizontalpodautoscaler="hpa1",metric_name="sqs_jobs",metric_target_type="value",namespace="ns1"} 30 - kube_horizontalpodautoscaler_spec_target_metric{horizontalpodautoscaler="hpa1",metric_name="transactions_processed",metric_target_type="average",namespace="ns1"} 33 + kube_horizontalpodautoscaler_spec_target_metric{horizontalpodautoscaler="hpa1",metric_name="sqs_jobs",metric_target_type="value",namespace="ns1",selectorlabel_app="foobar",selectorlabel_job_1_type="type1"} 30 + kube_horizontalpodautoscaler_spec_target_metric{horizontalpodautoscaler="hpa1",metric_name="transactions_processed",metric_target_type="average",namespace="ns1",selectorlabel_app="foobar",selectorlabel_k8s_io_name="foobar"} 33 kube_horizontalpodautoscaler_status_target_metric{horizontalpodautoscaler="hpa1",metric_name="cpu",metric_target_type="average",namespace="ns1"} 0.007 kube_horizontalpodautoscaler_status_target_metric{horizontalpodautoscaler="hpa1",metric_name="cpu",metric_target_type="utilization",namespace="ns1"} 80 kube_horizontalpodautoscaler_status_target_metric{horizontalpodautoscaler="hpa1",metric_name="memory",metric_target_type="average",namespace="ns1"} 2.6335914666e+07 @@ -293,6 +307,10 @@ func TestHPAStore(t *testing.T) { External: &autoscaling.ExternalMetricSource{ Metric: autoscaling.MetricIdentifier{ Name: "traefik_backend_requests_per_second", + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "foobar", + }}, }, Target: autoscaling.MetricTarget{ Value: resourcePtr(resource.MustParse("100")), @@ -363,6 +381,11 @@ func TestHPAStore(t *testing.T) { External: &autoscaling.ExternalMetricStatus{ Metric: autoscaling.MetricIdentifier{ Name: "traefik_backend_requests_per_second", + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "foobar", + "app-type": "traefik", + }}, }, Current: autoscaling.MetricValueStatus{ Value: resourcePtr(resource.MustParse("0")), @@ -393,15 +416,15 @@ func TestHPAStore(t *testing.T) { kube_horizontalpodautoscaler_spec_target_metric{horizontalpodautoscaler="hpa2",metric_name="cpu",metric_target_type="utilization",namespace="ns1"} 80 kube_horizontalpodautoscaler_spec_target_metric{horizontalpodautoscaler="hpa2",metric_name="memory",metric_target_type="utilization",namespace="ns1"} 75 kube_horizontalpodautoscaler_spec_target_metric{horizontalpodautoscaler="hpa2",metric_name="traefik_backend_errors_per_second",metric_target_type="value",namespace="ns1"} 100 - kube_horizontalpodautoscaler_spec_target_metric{horizontalpodautoscaler="hpa2",metric_name="traefik_backend_requests_per_second",metric_target_type="value",namespace="ns1"} 100 + kube_horizontalpodautoscaler_spec_target_metric{horizontalpodautoscaler="hpa2",metric_name="traefik_backend_requests_per_second",metric_target_type="value",namespace="ns1",selectorlabel_app="foobar"} 100 kube_horizontalpodautoscaler_status_target_metric{horizontalpodautoscaler="hpa2",metric_name="memory",metric_target_type="average",namespace="ns1"} 8.47775744e+08 kube_horizontalpodautoscaler_status_target_metric{horizontalpodautoscaler="hpa2",metric_name="memory",metric_target_type="utilization",namespace="ns1"} 28 kube_horizontalpodautoscaler_status_target_metric{horizontalpodautoscaler="hpa2",metric_name="cpu",metric_target_type="average",namespace="ns1"} 0.062 kube_horizontalpodautoscaler_status_target_metric{horizontalpodautoscaler="hpa2",metric_name="cpu",metric_target_type="utilization",namespace="ns1"} 6 kube_horizontalpodautoscaler_status_target_metric{horizontalpodautoscaler="hpa2",metric_name="cpu",metric_target_type="average",namespace="ns1"} 0.08 kube_horizontalpodautoscaler_status_target_metric{horizontalpodautoscaler="hpa2",metric_name="cpu",metric_target_type="utilization",namespace="ns1"} 10 - kube_horizontalpodautoscaler_status_target_metric{horizontalpodautoscaler="hpa2",metric_name="traefik_backend_requests_per_second",metric_target_type="value",namespace="ns1"} 0 - kube_horizontalpodautoscaler_status_target_metric{horizontalpodautoscaler="hpa2",metric_name="traefik_backend_requests_per_second",metric_target_type="average",namespace="ns1"} 2.9 + kube_horizontalpodautoscaler_status_target_metric{horizontalpodautoscaler="hpa2",metric_name="traefik_backend_requests_per_second",metric_target_type="value",namespace="ns1",selectorlabel_app="foobar",selectorlabel_app_type="traefik"} 0 + kube_horizontalpodautoscaler_status_target_metric{horizontalpodautoscaler="hpa2",metric_name="traefik_backend_requests_per_second",metric_target_type="average",namespace="ns1",selectorlabel_app="foobar",selectorlabel_app_type="traefik"} 2.9 kube_horizontalpodautoscaler_status_target_metric{horizontalpodautoscaler="hpa2",metric_name="traefik_backend_errors_per_second",metric_target_type="value",namespace="ns1"} 0 kube_horizontalpodautoscaler_status_condition{condition="AbleToScale",horizontalpodautoscaler="hpa2",namespace="ns1",status="false"} 0 kube_horizontalpodautoscaler_status_condition{condition="AbleToScale",horizontalpodautoscaler="hpa2",namespace="ns1",status="true"} 1