feat: rework instsance types to include zone/region

Signed-off-by: Dinar Valeev <k0da@opensuse.org>

Author: k0da

README.md

@@ -2,6 +2,8 @@
 
 Karpenter provider for VMWare Vsphere
 
+# !!!Early alpha - NOT for Production use!!!
+
 # Required flags
 
 | Flag             | Environment variable | Required |
@@ -22,7 +24,13 @@ Besides `VSPHERE_PATH` (vsphere folder to place virtulal machines on), all place
 
 All selectors have `tag` and `name` properties, those are mutually exclusive. Karpenter will find a resource either by Tag or Name.
 
-* `.spec.instanceTypes` - a list of desired instance types, in a following format: `%cpu-%mem-%os`, e.g. 8-16-linux
+* `.spec.instanceTypes` - a list of desired instance types:
+  - `os`: linux
+  - `cpu`: number of CPUS
+  - `memory`: amount of memory in gigabytes
+  - `region`: region topology
+  - `zone`: zone topology
+  - `maxPods`: maxPods to pass to kubelet (not implemented)
 * `.spec.diskSize` - a desired root volume size in Gigabytes
 
 * `.spec.tags` - a list of tags to apply to Karpenter managed virtual machines
@@ -36,4 +44,4 @@ All selectors have `tag` and `name` properties, those are mutually exclusive. Ka
   - `templateBase64` - A base64 encoded template
   - `values` - a v1.Secret reference (name/namespace) to key/values used in a template
 
-[!NOTE] user-data should hanndle `karpenter.sh/unregistered` taint to the node
+[!NOTE] user-data should hanndle `karpenter.sh/unregistered` taint to the node

charts/karpenter-crd/templates/karpenter.vsphere.com_vspherenodeclasses.yaml

@@ -119,7 +119,22 @@ spec:
                 type: object
               instanceTypes:
                 items:
-                  type: string
+                  properties:
+                    arch:
+                      type: string
+                    cpu:
+                      type: string
+                    maxPods:
+                      type: string
+                    memory:
+                      type: string
+                    os:
+                      type: string
+                    region:
+                      type: string
+                    zone:
+                      type: string
+                  type: object
                 type: array
               networkSelector:
                 properties:

charts/karpenter/crds/karpenter.vsphere.com_vspherenodeclasses.yaml

@@ -116,7 +116,22 @@ spec:
                 type: object
               instanceTypes:
                 items:
-                  type: string
+                  properties:
+                    arch:
+                      type: string
+                    cpu:
+                      type: string
+                    maxPods:
+                      type: string
+                    memory:
+                      type: string
+                    os:
+                      type: string
+                    region:
+                      type: string
+                    zone:
+                      type: string
+                  type: object
                 type: array
               networkSelector:
                 properties:

pkg/apis/crds/karpenter.vsphere.com_vspherenodeclasses.yaml

@@ -116,7 +116,22 @@ spec:
                 type: object
               instanceTypes:
                 items:
-                  type: string
+                  properties:
+                    arch:
+                      type: string
+                    cpu:
+                      type: string
+                    maxPods:
+                      type: string
+                    memory:
+                      type: string
+                    os:
+                      type: string
+                    region:
+                      type: string
+                    zone:
+                      type: string
+                  type: object
                 type: array
               networkSelector:
                 properties:

pkg/apis/v1alpha1/labels.go

@@ -2,6 +2,7 @@ package v1alpha1
 
 import (
 	"github.com/absaoss/karpenter-provider-vsphere/pkg/apis"
+	corev1 "k8s.io/api/core/v1"
 	coreapis "sigs.k8s.io/karpenter/pkg/apis"
 	karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1"
 )
@@ -23,6 +24,7 @@ var (
 	LabelInstanceCPU                      = apis.Group + "/instance-cpu"
 	LabelInstanceMemory                   = apis.Group + "/instance-memory"
 	LabelInstanceSize                     = apis.Group + "/instance-size"
+	LabelInstanceType                     = corev1.LabelInstanceTypeStable
 	AnnotationVsphereNodeClassHashVersion = apis.Group + "/vspherenodeclass-hash-version"
 	NodeClaimTagKey                       = coreapis.Group + "/nodeclaim"
 	NodePoolTagKey                        = karpv1.NodePoolLabelKey

pkg/apis/v1alpha1/vspherenodeclass.go

@@ -98,7 +98,7 @@ type VsphereNodeClassSpec struct {
 	Datacenter        DCSelectorTerm        `json:"dcSelector,omitempty"`
 	ImageSelector     ImageSelectorTerm     `json:"imageSelector,omitempty"`
 	DiskSize          int64                 `json:"diskSize,omitempty"`
-	InstanceTypes     []string              `json:"instanceTypes,omitempty"`
+	InstanceTypes     []InstanceType        `json:"instanceTypes,omitempty"`
 	UserData          UserData              `json:"userData,omitempty"`
 	Tags              map[string]string     `json:"tags,omitempty"`
 }
@@ -124,6 +124,8 @@ type InstanceType struct {
 	MaxPods string `json:"maxPods,omitempty"`
 	OS      string `json:"os,omitempty"`
 	Arch    string `json:"arch,omitempty"`
+	Zone    string `json:"zone,omitempty"`
+	Region  string `json:"region,omitempty"`
 }
 
 func (nc *VsphereNodeClass) Hash() string {

pkg/apis/v1alpha1/zz_generated.deepcopy.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	stderrors "errors"
 	"fmt"
-	"regexp"
 	"strings"
 	"time"
 
@@ -114,6 +113,8 @@ func (c *CloudProvider) instanceToNodeClaim(i *instance.Instance, instanceType *
 		nodeClaim.Status.Capacity = lo.PickBy(instanceType.Capacity, resourceFilter)
 		nodeClaim.Status.Allocatable = lo.PickBy(instanceType.Allocatable(), resourceFilter)
 	}
+	labels[corev1.LabelInstanceTypeStable] = instanceType.Name
+
 	//TODO: Figure out TopologyZone label
 	labels[corev1.LabelTopologyZone] = i.Tags[corev1.LabelTopologyZone]
 
@@ -277,22 +278,9 @@ func instanceTypesFromNodeClass(nodeClass *v1alpha1.VsphereNodeClass) []*cloudpr
 	instanceTypes := []*cloudprovider.InstanceType{}
 	var cpu, memory, os string
 	for _, t := range nodeClass.Spec.InstanceTypes {
-		instanceTypeRegex := regexp.MustCompile(`(?P<CPU>.*)-(?P<Memory>.*)-(?P<OS>.*)`)
-		matches := instanceTypeRegex.FindStringSubmatch(t)
-		if matches == nil {
-			fmt.Printf("instance type %s does not match expected format\n", t)
-		}
-		for i, name := range instanceTypeRegex.SubexpNames() {
-			if name == "CPU" {
-				cpu = matches[i]
-			}
-			if name == "Memory" {
-				memory = matches[i]
-			}
-			if name == "OS" {
-				os = matches[i]
-			}
-		}
+		cpu = t.CPU
+		memory = t.Memory
+		os = strings.ToLower(t.OS)
 		typeName := fmt.Sprintf("vsphere-vm.cpu-%s.mem-%sgb.os-%s", cpu, memory, os)
 		instanceType := &cloudprovider.InstanceType{
 			Name: typeName,
@@ -304,15 +292,15 @@ func instanceTypesFromNodeClass(nodeClass *v1alpha1.VsphereNodeClass) []*cloudpr
 			Capacity: corev1.ResourceList{
 				corev1.ResourceCPU:              resource.MustParse(cpu),
 				corev1.ResourceMemory:           resource.MustParse(memory),
-				corev1.ResourcePods:             resource.MustParse("110"),
+				corev1.ResourcePods:             resource.MustParse(t.MaxPods),
 				corev1.ResourceEphemeralStorage: resource.MustParse(utils.GiToByteAsString(nodeClass.Spec.DiskSize)),
 			},
 			//TODO: compute kubelet overhead
 			Overhead: &cloudprovider.InstanceTypeOverhead{},
 			Offerings: []*cloudprovider.Offering{
 				{
 					Requirements: scheduling.NewRequirements(
-						scheduling.NewRequirement(corev1.LabelTopologyZone, corev1.NodeSelectorOpIn)),
+						scheduling.NewRequirement(corev1.LabelTopologyZone, corev1.NodeSelectorOpIn, t.Zone)),
 					Price:     float64(0.0),
 					Available: true,
 				},

pkg/cloudprovider/cloudprovider.go

@@ -77,8 +77,6 @@ func (c *Controller) updateNodeClaimHash(ctx context.Context, nodeClass *v1alpha
 				v1alpha1.AnnotationVsphereNodeClassHashVersion: v1alpha1.VsphereNodeClassHashVersion,
 			})
 
-			// Any NodeClaim that is already drifted will remain drifted if the karpenter.azure.com/nodepool-hash-version doesn't match
-			// Since the hashing mechanism has changed we will not be able to determine if the drifted status of the NodeClaim has changed
 			if nc.StatusConditions().Get(karpv1.ConditionTypeDrifted) == nil {
 				nc.Annotations = lo.Assign(nc.Annotations, map[string]string{
 					v1alpha1.AnnotationVsphereNodeClassHash: nodeClass.Hash(),