Skip to content

Commit 6e28c43

Browse files
Create a separate controller for generating availability metrics
1 parent 17444af commit 6e28c43

File tree

12 files changed

+787
-190
lines changed

12 files changed

+787
-190
lines changed

hack/code/instancetype_testdata_gen/main.go

Lines changed: 40 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,9 @@ import (
2828
"github.com/aws/aws-sdk-go-v2/config"
2929
"github.com/aws/aws-sdk-go-v2/service/ec2"
3030
ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types"
31-
sdk "github.com/aws/karpenter-provider-aws/pkg/aws"
3231
"github.com/samber/lo"
32+
33+
sdk "github.com/aws/karpenter-provider-aws/pkg/aws"
3334
)
3435

3536
const packageHeader = `
@@ -39,6 +40,7 @@ import (
3940
"github.com/aws/aws-sdk-go-v2/aws"
4041
"github.com/aws/aws-sdk-go-v2/service/ec2"
4142
ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types"
43+
"github.com/samber/lo"
4244
)
4345
4446
// GENERATED FILE. DO NOT EDIT DIRECTLY.
@@ -74,6 +76,7 @@ func main() {
7476
fmt.Fprintln(src, string(license))
7577
fmt.Fprint(src, packageHeader)
7678
fmt.Fprintln(src, getDescribeInstanceTypesOutput(ctx, ec2api, instanceTypes))
79+
fmt.Fprintln(src, getDescribeInstanceTypeOfferingsOutput())
7780

7881
// Format and print to the file
7982
formatted, err := format.Source(src.Bytes())
@@ -116,6 +119,42 @@ func getDescribeInstanceTypesOutput(ctx context.Context, ec2api sdk.EC2API, inst
116119
return src.String()
117120
}
118121

122+
func getDescribeInstanceTypeOfferingsOutput() string {
123+
src := &bytes.Buffer{}
124+
instanceTypeToZones := map[string][]string{
125+
"m5.large": {"test-zone-1a", "test-zone-1b", "test-zone-1c", "test-zone-1a-local"},
126+
"m5.xlarge": {"test-zone-1a", "test-zone-1b"},
127+
"m5.2xlarge": {"test-zone-1a"},
128+
"m5.4xlarge": {"test-zone-1a"},
129+
"m5.8xlarge": {"test-zone-1a"},
130+
"p3.8xlarge": {"test-zone-1a", "test-zone-1b"},
131+
"dl1.24xlarge": {"test-zone-1a", "test-zone-1b"},
132+
"g4dn.8xlarge": {"test-zone-1a", "test-zone-1b"},
133+
"g4ad.16xlarge": {"test-zone-1a", "test-zone-1b"},
134+
"t3.large": {"test-zone-1a", "test-zone-1b"},
135+
"inf2.xlarge": {"test-zone-1a"},
136+
"inf2.24xlarge": {"test-zone-1a"},
137+
"trn1.2xlarge": {"test-zone-1a"},
138+
"c6g.large": {"test-zone-1a"},
139+
"m5.metal": {"test-zone-1a", "test-zone-1b"},
140+
"m6idn.32xlarge": {"test-zone-1a", "test-zone-1b", "test-zone-1c"},
141+
}
142+
143+
fmt.Fprintln(src, "var defaultDescribeInstanceTypeOfferingsOutput = &ec2.DescribeInstanceTypeOfferingsOutput{")
144+
fmt.Fprintln(src, "InstanceTypeOfferings: []ec2types.InstanceTypeOffering{")
145+
for _, elem := range lo.Flatten(lo.MapToSlice(instanceTypeToZones, func(it string, zones []string) []lo.Tuple2[string, string] {
146+
return lo.Map(zones, func(z string, _ int) lo.Tuple2[string, string] { return lo.Tuple2[string, string]{A: it, B: z} })
147+
})) {
148+
fmt.Fprintln(src, "{")
149+
fmt.Fprintf(src, "InstanceType: \"%s\",\n", elem.A)
150+
fmt.Fprintf(src, "Location: lo.ToPtr(\"%s\"),\n", elem.B)
151+
fmt.Fprintln(src, "},")
152+
}
153+
fmt.Fprintln(src, "},")
154+
fmt.Fprintln(src, "}")
155+
return src.String()
156+
}
157+
119158
func getInstanceTypeInfo(info ec2types.InstanceTypeInfo) string {
120159
src := &bytes.Buffer{}
121160

pkg/controllers/controllers.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import (
2727

2828
v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1"
2929
sdk "github.com/aws/karpenter-provider-aws/pkg/aws"
30+
"github.com/aws/karpenter-provider-aws/pkg/controllers/metrics"
3031
"github.com/aws/karpenter-provider-aws/pkg/controllers/nodeclass"
3132
nodeclasshash "github.com/aws/karpenter-provider-aws/pkg/controllers/nodeclass/hash"
3233
controllersinstancetype "github.com/aws/karpenter-provider-aws/pkg/controllers/providers/instancetype"
@@ -97,6 +98,7 @@ func NewControllers(
9798
status.NewController[*v1.EC2NodeClass](kubeClient, mgr.GetEventRecorderFor("karpenter"), status.EmitDeprecatedMetrics),
9899
controllersversion.NewController(versionProvider, versionProvider.UpdateVersionWithValidation),
99100
capacityreservation.NewController(kubeClient, cloudProvider),
101+
metrics.NewController(kubeClient, cloudProvider),
100102
}
101103
if options.FromContext(ctx).InterruptionQueue != "" {
102104
sqsapi := servicesqs.NewFromConfig(cfg)
Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
/*
2+
Licensed under the Apache License, Version 2.0 (the "License");
3+
you may not use this file except in compliance with the License.
4+
You may obtain a copy of the License at
5+
6+
http://www.apache.org/licenses/LICENSE-2.0
7+
8+
Unless required by applicable law or agreed to in writing, software
9+
distributed under the License is distributed on an "AS IS" BASIS,
10+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
See the License for the specific language governing permissions and
12+
limitations under the License.
13+
*/
14+
15+
package metrics
16+
17+
import (
18+
"context"
19+
"time"
20+
21+
"github.com/awslabs/operatorpkg/singleton"
22+
"github.com/samber/lo"
23+
"k8s.io/apimachinery/pkg/util/sets"
24+
controllerruntime "sigs.k8s.io/controller-runtime"
25+
"sigs.k8s.io/controller-runtime/pkg/client"
26+
"sigs.k8s.io/controller-runtime/pkg/manager"
27+
"sigs.k8s.io/controller-runtime/pkg/reconcile"
28+
karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1"
29+
"sigs.k8s.io/karpenter/pkg/cloudprovider"
30+
)
31+
32+
type metricDimensions struct {
33+
instanceType string
34+
capacityType string
35+
zone string
36+
}
37+
38+
type Controller struct {
39+
kubeClient client.Client
40+
cloudProvider cloudprovider.CloudProvider
41+
}
42+
43+
func NewController(kubeClient client.Client, cloudProvider cloudprovider.CloudProvider) *Controller {
44+
return &Controller{
45+
kubeClient: kubeClient,
46+
cloudProvider: cloudProvider,
47+
}
48+
}
49+
50+
func (c *Controller) Reconcile(ctx context.Context) (reconcile.Result, error) {
51+
nodePools := &karpv1.NodePoolList{}
52+
if err := c.kubeClient.List(ctx, nodePools); err != nil {
53+
return reconcile.Result{}, err
54+
}
55+
availability := map[metricDimensions]bool{}
56+
price := map[metricDimensions]float64{}
57+
for _, nodePool := range nodePools.Items {
58+
instanceTypes, err := c.cloudProvider.GetInstanceTypes(ctx, &nodePool)
59+
if err != nil {
60+
return reconcile.Result{}, err
61+
}
62+
for _, instanceType := range instanceTypes {
63+
zones := sets.New[string]()
64+
for _, offering := range instanceType.Offerings {
65+
dimensions := metricDimensions{instanceType: instanceType.Name, capacityType: offering.CapacityType(), zone: offering.Zone()}
66+
availability[dimensions] = availability[dimensions] || offering.Available
67+
price[dimensions] = offering.Price
68+
zones.Insert(offering.Zone())
69+
}
70+
for zone := range zones {
71+
dimensions := metricDimensions{instanceType: instanceType.Name, capacityType: karpv1.CapacityTypeReserved, zone: zone}
72+
if _, ok := availability[dimensions]; !ok {
73+
availability[dimensions] = false
74+
price[dimensions] = 0
75+
}
76+
}
77+
}
78+
}
79+
80+
for dimensions, available := range availability {
81+
InstanceTypeOfferingAvailable.Set(float64(lo.Ternary(available, 1, 0)), map[string]string{
82+
instanceTypeLabel: dimensions.instanceType,
83+
capacityTypeLabel: dimensions.capacityType,
84+
zoneLabel: dimensions.zone,
85+
})
86+
}
87+
for dimensions, p := range price {
88+
InstanceTypeOfferingPriceEstimate.Set(p, map[string]string{
89+
instanceTypeLabel: dimensions.instanceType,
90+
capacityTypeLabel: dimensions.capacityType,
91+
zoneLabel: dimensions.zone,
92+
})
93+
}
94+
return reconcile.Result{RequeueAfter: time.Minute}, nil
95+
}
96+
97+
func (c *Controller) Register(_ context.Context, m manager.Manager) error {
98+
return controllerruntime.NewControllerManagedBy(m).
99+
Named("cloudprovider.metrics").
100+
WatchesRawSource(singleton.Source()).
101+
Complete(singleton.AsReconciler(c))
102+
}

pkg/providers/instancetype/offering/metrics.go renamed to pkg/controllers/metrics/metrics.go

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,13 +12,12 @@ See the License for the specific language governing permissions and
1212
limitations under the License.
1313
*/
1414

15-
package offering
15+
package metrics
1616

1717
import (
1818
opmetrics "github.com/awslabs/operatorpkg/metrics"
1919
"github.com/prometheus/client_golang/prometheus"
2020
crmetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
21-
2221
"sigs.k8s.io/karpenter/pkg/metrics"
2322
)
2423

0 commit comments

Comments
 (0)