Create a separate controller for generating availability metrics

Author: jonathan-innis

pkg/controllers/controllers.go

@@ -27,6 +27,7 @@ import (
 
 	v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1"
 	sdk "github.com/aws/karpenter-provider-aws/pkg/aws"
+	"github.com/aws/karpenter-provider-aws/pkg/controllers/metrics"
 	"github.com/aws/karpenter-provider-aws/pkg/controllers/nodeclass"
 	nodeclasshash "github.com/aws/karpenter-provider-aws/pkg/controllers/nodeclass/hash"
 	controllersinstancetype "github.com/aws/karpenter-provider-aws/pkg/controllers/providers/instancetype"
@@ -97,6 +98,7 @@ func NewControllers(
 		status.NewController[*v1.EC2NodeClass](kubeClient, mgr.GetEventRecorderFor("karpenter"), status.EmitDeprecatedMetrics),
 		controllersversion.NewController(versionProvider, versionProvider.UpdateVersionWithValidation),
 		capacityreservation.NewController(kubeClient, cloudProvider),
+		metrics.NewController(kubeClient, cloudProvider),
 	}
 	if options.FromContext(ctx).InterruptionQueue != "" {
 		sqsapi := servicesqs.NewFromConfig(cfg)

pkg/controllers/metrics/controller.go

@@ -0,0 +1,102 @@
+/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metrics
+
+import (
+	"context"
+	"time"
+
+	"github.com/awslabs/operatorpkg/singleton"
+	"github.com/samber/lo"
+	"k8s.io/apimachinery/pkg/util/sets"
+	controllerruntime "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1"
+	"sigs.k8s.io/karpenter/pkg/cloudprovider"
+)
+
+type metricDimensions struct {
+	instanceType string
+	capacityType string
+	zone         string
+}
+
+type Controller struct {
+	kubeClient    client.Client
+	cloudProvider cloudprovider.CloudProvider
+}
+
+func NewController(kubeClient client.Client, cloudProvider cloudprovider.CloudProvider) *Controller {
+	return &Controller{
+		kubeClient:    kubeClient,
+		cloudProvider: cloudProvider,
+	}
+}
+
+func (c *Controller) Reconcile(ctx context.Context) (reconcile.Result, error) {
+	nodePools := &karpv1.NodePoolList{}
+	if err := c.kubeClient.List(ctx, nodePools); err != nil {
+		return reconcile.Result{}, err
+	}
+	availability := map[metricDimensions]bool{}
+	price := map[metricDimensions]float64{}
+	for _, nodePool := range nodePools.Items {
+		instanceTypes, err := c.cloudProvider.GetInstanceTypes(ctx, &nodePool)
+		if err != nil {
+			return reconcile.Result{}, err
+		}
+		for _, instanceType := range instanceTypes {
+			zones := sets.New[string]()
+			for _, offering := range instanceType.Offerings {
+				dimensions := metricDimensions{instanceType: instanceType.Name, capacityType: offering.CapacityType(), zone: offering.Zone()}
+				availability[dimensions] = availability[dimensions] || offering.Available
+				price[dimensions] = offering.Price
+				zones.Insert(offering.Zone())
+			}
+			for zone := range zones {
+				dimensions := metricDimensions{instanceType: instanceType.Name, capacityType: karpv1.CapacityTypeReserved, zone: zone}
+				if _, ok := availability[dimensions]; !ok {
+					availability[dimensions] = false
+					price[dimensions] = 0
+				}
+			}
+		}
+	}
+
+	for dimensions, available := range availability {
+		InstanceTypeOfferingAvailable.Set(float64(lo.Ternary(available, 1, 0)), map[string]string{
+			instanceTypeLabel: dimensions.instanceType,
+			capacityTypeLabel: dimensions.capacityType,
+			zoneLabel:         dimensions.zone,
+		})
+	}
+	for dimensions, p := range price {
+		InstanceTypeOfferingPriceEstimate.Set(p, map[string]string{
+			instanceTypeLabel: dimensions.instanceType,
+			capacityTypeLabel: dimensions.capacityType,
+			zoneLabel:         dimensions.zone,
+		})
+	}
+	return reconcile.Result{RequeueAfter: time.Minute}, nil
+}
+
+func (c *Controller) Register(_ context.Context, m manager.Manager) error {
+	return controllerruntime.NewControllerManagedBy(m).
+		Named("cloudprovider.metrics").
+		WatchesRawSource(singleton.Source()).
+		Complete(singleton.AsReconciler(c))
+}

…oviders/instancetype/offering/metrics.go pkg/controllers/metrics/metrics.gopkg/providers/instancetype/offering/metrics.go renamed to pkg/controllers/metrics/metrics.go pkg/providers/instancetype/offering/metrics.go renamed to pkg/controllers/metrics/metrics.go

@@ -12,13 +12,12 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package offering
+package metrics
 
 import (
 	opmetrics "github.com/awslabs/operatorpkg/metrics"
 	"github.com/prometheus/client_golang/prometheus"
 	crmetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
-
 	"sigs.k8s.io/karpenter/pkg/metrics"
 )
 

pkg/providers/instancetype/metrics.go

@@ -25,8 +25,6 @@ import (
 const (
 	cloudProviderSubsystem = "cloudprovider"
 	instanceTypeLabel      = "instance_type"
-	capacityTypeLabel      = "capacity_type"
-	zoneLabel              = "zone"
 )
 
 var (

…viders/instancetype/offering/provider.go …viders/instancetype/offering/offering.gopkg/providers/instancetype/offering/provider.go renamed to pkg/providers/instancetype/offering/offering.go pkg/providers/instancetype/offering/provider.go renamed to pkg/providers/instancetype/offering/offering.go

@@ -78,34 +78,6 @@ func (p *DefaultProvider) InjectOfferings(
 			allZones,
 			subnetZones,
 		)
-
-		reservedAvailability := map[string]bool{}
-		for _, of := range offerings {
-			// If the capacity type is reserved we need to determine if any of the reserved offerings are available. Otherwise,
-			// we can update the availability metric directly.
-			if of.CapacityType() == karpv1.CapacityTypeReserved {
-				reservedAvailability[of.Zone()] = reservedAvailability[of.Zone()] || of.Available
-			} else {
-				InstanceTypeOfferingAvailable.Set(float64(lo.Ternary(of.Available, 1, 0)), map[string]string{
-					instanceTypeLabel: it.Name,
-					capacityTypeLabel: of.Requirements.Get(karpv1.CapacityTypeLabelKey).Any(),
-					zoneLabel:         of.Requirements.Get(corev1.LabelTopologyZone).Any(),
-				})
-			}
-			InstanceTypeOfferingPriceEstimate.Set(of.Price, map[string]string{
-				instanceTypeLabel: it.Name,
-				capacityTypeLabel: of.Requirements.Get(karpv1.CapacityTypeLabelKey).Any(),
-				zoneLabel:         of.Requirements.Get(corev1.LabelTopologyZone).Any(),
-			})
-		}
-		for zone := range allZones {
-			InstanceTypeOfferingAvailable.Set(float64(lo.Ternary(reservedAvailability[zone], 1, 0)), map[string]string{
-				instanceTypeLabel: it.Name,
-				capacityTypeLabel: karpv1.CapacityTypeReserved,
-				zoneLabel:         zone,
-			})
-		}
-
 		// NOTE: By making this copy one level deep, we can modify the offerings without mutating the results from previous
 		// GetInstanceTypes calls. This should still be done with caution - it is currently done here in the provider, and
 		// once in the instance provider (filterReservedInstanceTypes)