chore: update README and minor cleanups

k0da · k0da · commit 18e1db7e52a9 · 2025-06-12T13:17:13.000+02:00
Signed-off-by: Dinar Valeev &lt;k0da@opensuse.org&gt;
diff --git a/README.md b/README.md
@@ -1 +1,40 @@
-karpenter-provider-vsphere
+# karpenter-provider-vsphere
+
+Karpenter provider for VMWare Vsphere
+
+# Required flags
+
+| Flag             | Environment variable | Required |
+|------------------|----------------------|----------|
+|cluster-name      | CLUSTER_NAME         | true     |
+| vsphere-endpoint | GOVC_URL             | true     |
+| vsphere-username | GOVC_USERNAME        | true     |
+| vsphere-password | GOVC_PASSWORD        | true     |
+| vsphere-path     | VSPHERE_PATH         | true     |
+| vsphere-insecure | VSPHERE_INSECURE     | false    |
+
+# VsphereNodeClass API
+Besides `VSPHERE_PATH` (vsphere folder to place virtulal machines on), all placement settings are defined in `VsphereNodeClass` resource. This is done via selectors:
+* `.spec.computeSelector` - defines how to search for desired resourcePool
+* `.spec.datastoreSelector` - defines how to search for desired datastore
+* `.spec.networkSelector` - difines how to discover network
+* `.spec.imageSelector` - VM Template to use for VM Clone
+
+All selectors have `tag` and `name` properties, those are mutually exclusive. Karpenter will find a resource either by Tag or Name.
+[!NOTE]
+Due to nature of Vsphere tags. Only 1 uniquie tag could be selected to identify desired resource
+
+
+* `.spec.instanceTypes` - a list of desired instance types, in a following format: `%cpu-%mem-%os`, e.g. 8-16-linux
+* `.spec.diskSize` - a desired root volume size in Gigabytes
+
+* `.spec.tags` - a list of tags to apply to Karpenter managed virtual machines
+  [!NOTE]
+  At least two tags must be specified explicitly:
+  * `topology.kubernetes.io/zone` and `k8s-zone` to satisfy Vsphere Cloud controller manager which bootstraps Kubernetes node and removes `unitialized` Taint.
+
+
+* `.spec.userdata`:
+  - `type` - Either `ignition` or `cloud-init`
+  - `templateBase64` - A base64 encoded template
+  - `values` - a v1.Secret reference (name/namespace) to key/values used in a template
diff --git a/charts/karpenter-crd/templates/karpenter.vsphere.com_vspherenodeclasses.yaml b/charts/karpenter-crd/templates/karpenter.vsphere.com_vspherenodeclasses.yaml
@@ -138,6 +138,10 @@ spec:
                     - message: empty tag keys or values aren't supported
                       rule: self.all(k, k != '' && self[k] != '')
                 type: object
+              tags:
+                additionalProperties:
+                  type: string
+                type: object
               userData:
                 properties:
                   templateBase64:
diff --git a/charts/karpenter/crds/karpenter.vsphere.com_vspherenodeclasses.yaml b/charts/karpenter/crds/karpenter.vsphere.com_vspherenodeclasses.yaml
@@ -135,6 +135,10 @@ spec:
                     - message: empty tag keys or values aren't supported
                       rule: self.all(k, k != '' && self[k] != '')
                 type: object
+              tags:
+                additionalProperties:
+                  type: string
+                type: object
               userData:
                 properties:
                   templateBase64:
diff --git a/charts/karpenter/templates/deployment.yaml b/charts/karpenter/templates/deployment.yaml
@@ -89,8 +89,6 @@ spec:
           image: {{ include "karpenter.controller.image" . }}
           imagePullPolicy: {{ .Values.imagePullPolicy }}
           env:
-            - name: KUBERNETES_MIN_VERSION
-              value: "1.19.0-0"
             - name: KARPENTER_SERVICE
               value: {{ include "karpenter.fullname" . }}
           {{- with .Values.logLevel }}
@@ -130,37 +128,9 @@ spec:
           {{- with .Values.settings.preferencePolicy }}
             - name: PREFERENCE_POLICY
               value: "{{ . }}"
-          {{- end }}
-          {{- with .Values.settings.clusterCABundle }}
-            - name: CLUSTER_CA_BUNDLE
-              value: "{{ tpl (toString .) $ }}"
           {{- end }}
             - name: CLUSTER_NAME
               value: "{{ required "Chart cannot be installed without a valid settings.clusterName!" (tpl .Values.settings.clusterName .) }}"
-          {{- with .Values.settings.clusterEndpoint }}
-            - name: CLUSTER_ENDPOINT
-              value: "{{ tpl (toString .) $ }}"
-          {{- end }}
-          {{- with .Values.settings.isolatedVPC }}
-            - name: ISOLATED_VPC
-              value: "{{ tpl (toString .) $ }}"
-          {{- end }}
-          {{- with .Values.settings.eksControlPlane }}
-            - name: EKS_CONTROL_PLANE
-              value: "{{ tpl (toString .) $ }}"
-          {{- end }}
-          {{- with .Values.settings.vmMemoryOverheadPercent }}
-            - name: VM_MEMORY_OVERHEAD_PERCENT
-              value: "{{ tpl (toString .) $ }}"
-          {{- end }}
-          {{- with .Values.settings.interruptionQueue }}
-            - name: INTERRUPTION_QUEUE
-              value: "{{ tpl (toString .) $ }}"
-          {{- end }}
-          {{- with .Values.settings.reservedENIs }}
-            - name: RESERVED_ENIS
-              value: "{{ tpl (toString .) $ }}"
-          {{- end }}
           {{- with .Values.controller.env }}
             {{- toYaml . | nindent 12 }}
           {{- end }}
diff --git a/pkg/apis/crds/karpenter.vsphere.com_vspherenodeclasses.yaml b/pkg/apis/crds/karpenter.vsphere.com_vspherenodeclasses.yaml
@@ -135,6 +135,10 @@ spec:
                     - message: empty tag keys or values aren't supported
                       rule: self.all(k, k != '' && self[k] != '')
                 type: object
+              tags:
+                additionalProperties:
+                  type: string
+                type: object
               userData:
                 properties:
                   templateBase64:
diff --git a/pkg/apis/v1alpha1/vspherenodeclass.go b/pkg/apis/v1alpha1/vspherenodeclass.go
@@ -100,6 +100,7 @@ type VsphereNodeClassSpec struct {
 	DiskSize          int64                 `json:"diskSize,omitempty"`
 	InstanceTypes     []string              `json:"instanceTypes,omitempty"`
 	UserData          UserData              `json:"userData,omitempty"`
+	Tags              map[string]string     `json:"tags,omitempty"`
 }
 
 type UserDataType string
diff --git a/pkg/apis/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/v1alpha1/zz_generated.deepcopy.go
diff --git a/pkg/cloudprovider/cloudprovider.go b/pkg/cloudprovider/cloudprovider.go
@@ -89,8 +89,7 @@ func (c *CloudProvider) Create(ctx context.Context, nodeClaim *karpv1.NodeClaim)
 	if len(instanceTypes) == 0 {
 		return nil, cloudprovider.NewInsufficientCapacityError(fmt.Errorf("all requested instance types were unavailable during launch"))
 	}
-	nodePoolName := nodeClaim.Labels[karpv1.NodePoolLabelKey]
-	instance, err := c.instanceProvider.Create(ctx, nodeClass, nodeClaim, instanceTypes, nodePoolName)
+	instance, err := c.instanceProvider.Create(ctx, nodeClass, nodeClaim, instanceTypes)
 	if err != nil {
 		return nil, cloudprovider.NewCreateError(fmt.Errorf("creating instance failed, %w", err), CreateInstanceFailedReason, err.Error())
 	}
diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go
@@ -62,7 +62,6 @@ func NewOperator(ctx context.Context, operator *operator.Operator) (context.Cont
 		inClusterClient,
 		finderProvider,
 		options.FromContext(ctx).ClusterName,
-		options.FromContext(ctx).VsphereZone,
 	)
 	return ctx, &Operator{
 		Operator:                     operator,
diff --git a/pkg/providers/finder/tags.go b/pkg/providers/finder/tags.go
@@ -8,6 +8,7 @@ import (
 	"github.com/vmware/govmomi/vapi/tags"
 	"github.com/vmware/govmomi/vim25/types"
 	corev1 "k8s.io/api/core/v1"
+	"sigs.k8s.io/controller-runtime/pkg/log"
 )
 
 func (t *Provider) getObjectByType(ctx context.Context, tag *tags.Tag, objT string) (*types.ManagedObjectReference, error) {
@@ -164,7 +165,7 @@ func getCategoryObject(name string) *tags.Category {
 func (t *Provider) TagsFromVM(ctx context.Context, vm *object.VirtualMachine) (map[string]string, error) {
 	tagsAttached, err := t.TagManager.ListAttachedTags(ctx, vm.Reference())
 	if err != nil {
-		fmt.Printf("Failed to list tags for VM %s: %v\n", vm.Name(), err)
+		log.FromContext(ctx).Error(err, fmt.Sprintf("failed to list tags for VM %s", vm.Name()))
 	}
 	return extractTagInfo(ctx, t.TagManager, tagsAttached)
 
diff --git a/pkg/providers/instance/instance.go b/pkg/providers/instance/instance.go
@@ -3,6 +3,7 @@ package instance
 import (
 	"context"
 	"fmt"
+	"maps"
 	"strings"
 	"time"
 
@@ -12,14 +13,14 @@ import (
 	"github.com/vmware/govmomi/object"
 	models "github.com/vmware/govmomi/vim25/mo"
 	"github.com/vmware/govmomi/vim25/types"
-	corev1 "k8s.io/api/core/v1"
 	"k8s.io/client-go/kubernetes"
+	"sigs.k8s.io/controller-runtime/pkg/log"
 	karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1"
 	corecloudprovider "sigs.k8s.io/karpenter/pkg/cloudprovider"
 )
 
 type Provider interface {
-	Create(context.Context, *v1alpha1.VsphereNodeClass, *karpv1.NodeClaim, []*corecloudprovider.InstanceType, string) (*Instance, error)
+	Create(context.Context, *v1alpha1.VsphereNodeClass, *karpv1.NodeClaim, []*corecloudprovider.InstanceType) (*Instance, error)
 	Get(context.Context, string) (*Instance, error)
 	List(context.Context) ([]*Instance, error)
 	Delete(context.Context, string) error
@@ -29,15 +30,13 @@ var _ Provider = (*DefaultProvider)(nil)
 
 type DefaultProvider struct {
 	ClusterName string
-	VsphereZone string
 	kubeClient  kubernetes.Interface
 	Finder      *finder.Provider
 }
 
-func NewDefaultProvider(kube kubernetes.Interface, finder *finder.Provider, clusterName, zone string) *DefaultProvider {
+func NewDefaultProvider(kube kubernetes.Interface, finder *finder.Provider, clusterName string) *DefaultProvider {
 	return &DefaultProvider{
 		ClusterName: clusterName,
-		VsphereZone: zone,
 		kubeClient:  kube,
 		Finder:      finder,
 	}
@@ -107,20 +106,21 @@ func (p *DefaultProvider) Create(
 	ctx context.Context,
 	class *v1alpha1.VsphereNodeClass,
 	claim *karpv1.NodeClaim,
-	instanceTypes []*corecloudprovider.InstanceType, poolName string) (*Instance, error) {
+	instanceTypes []*corecloudprovider.InstanceType) (*Instance, error) {
 
 	instanceType := instanceTypes[0] // For simplicity, we take the first instance type.
 	VMName := GenerateVMName(p.ClusterName, claim.Name)
 	instanceTags := map[string]string{
 		v1alpha1.ClusterNameTagKey:   p.ClusterName,
 		v1alpha1.LabelNodeClass:      class.Name,
-		karpv1.NodePoolLabelKey:      poolName,
-		corev1.LabelTopologyZone:     p.VsphereZone,
+		karpv1.NodePoolLabelKey:      claim.Labels[karpv1.NodePoolLabelKey],
 		v1alpha1.LabelInstanceSize:   instanceType.Name,
 		v1alpha1.LabelInstanceCPU:    fmt.Sprintf("%d", instanceType.Capacity.Cpu().Value()),
 		v1alpha1.LabelInstanceMemory: fmt.Sprintf("%d", utils.GiToMb(instanceType.Capacity.Memory().ToDec().Value())),
 	}
 
+	maps.Copy(instanceTags, class.Spec.Tags)
+
 	cloneSpec, err := p.GenerateVMSpec(ctx, class, VMName, instanceType)
 	if err != nil {
 		return nil, fmt.Errorf("failed to generate VM spec: %w", err)
@@ -196,21 +196,21 @@ func (p *DefaultProvider) List(ctx context.Context) ([]*Instance, error) {
 	vms, err := p.Finder.ListVMs(ctx)
 	//
 	if err != nil {
-		fmt.Printf("Failed to list VMs: %v\n", err)
+		log.FromContext(ctx).Error(err, "")
 	}
 	for _, vm := range vms {
 		image := getImageFromAnnotation(vm)
 		tags, err := p.Finder.TagsFromVM(ctx, vm)
 		if err != nil {
-			fmt.Printf("Failed to get tags for VM %s: %v\n", vm.Name(), err)
+			log.FromContext(ctx).Error(err, fmt.Sprintf("failed to get tags for VM %s", vm.Name()))
 		}
 		ps, err := vm.PowerState(ctx)
 		if err != nil {
-			fmt.Printf("Failed to get power state for VM %s: %v\n", vm.Name(), err)
+			log.FromContext(ctx).Error(err, fmt.Sprintf("failed to get power state for VM %s", vm.Name()))
 		}
 		creationDate, err := extractCreationDate(ctx, vm)
 		if err != nil {
-			fmt.Printf("Failed to extract creation date for VM %s: %v\n", vm.Name(), err)
+			log.FromContext(ctx).Error(err, fmt.Sprintf("failed to extract creation date for VM %s", vm.Name()))
 		}
 		instances = append(instances, NewInstance(vm, vm.UUID(ctx), image, string(ps), vm.Name(), *creationDate, tags))
 	}
@@ -224,7 +224,7 @@ func (p *DefaultProvider) Get(ctx context.Context, vmID string) (*Instance, erro
 	}
 	tags, err := p.Finder.TagsFromVM(ctx, vm)
 	if err != nil {
-		fmt.Printf("Failed to get tags for VM %s: %v\n", vm.Name(), err)
+		log.FromContext(ctx).Error(err, fmt.Sprintf("failed to get tags for VM %s", vm.Name()))
 	}
 	instance := NewInstanceFromVM(ctx, vm, time.Now(), tags)
 	return instance, nil

Original file line number	Diff line number	Diff line change
`@@ -100,6 +100,7 @@ type VsphereNodeClassSpec struct {`
`100`	`100`	DiskSize int64 `json:"diskSize,omitempty"`
`101`	`101`	InstanceTypes []string `json:"instanceTypes,omitempty"`
`102`	`102`	UserData UserData `json:"userData,omitempty"`
	`103`	+ Tags map[string]string `json:"tags,omitempty"`
`103`	`104`	`}`
`104`	`105`
`105`	`106`	`type UserDataType string`
Original file line number	Diff line number	Diff line change
`@@ -89,8 +89,7 @@ func (c CloudProvider) Create(ctx context.Context, nodeClaim karpv1.NodeClaim)`
`89`	`89`	`if len(instanceTypes) == 0 {`
`90`	`90`	`return nil, cloudprovider.NewInsufficientCapacityError(fmt.Errorf("all requested instance types were unavailable during launch"))`
`91`	`91`	`}`
`92`		`- nodePoolName := nodeClaim.Labels[karpv1.NodePoolLabelKey]`
`93`		`- instance, err := c.instanceProvider.Create(ctx, nodeClass, nodeClaim, instanceTypes, nodePoolName)`
	`92`	`+ instance, err := c.instanceProvider.Create(ctx, nodeClass, nodeClaim, instanceTypes)`
`94`	`93`	`if err != nil {`
`95`	`94`	`return nil, cloudprovider.NewCreateError(fmt.Errorf("creating instance failed, %w", err), CreateInstanceFailedReason, err.Error())`
`96`	`95`	`}`
Original file line number	Diff line number	Diff line change
`@@ -62,7 +62,6 @@ func NewOperator(ctx context.Context, operator *operator.Operator) (context.Cont`
`62`	`62`	`inClusterClient,`
`63`	`63`	`finderProvider,`
`64`	`64`	`options.FromContext(ctx).ClusterName,`
`65`		`- options.FromContext(ctx).VsphereZone,`
`66`	`65`	`)`
`67`	`66`	`return ctx, &Operator{`
`68`	`67`	`Operator: operator,`
Original file line number	Diff line number	Diff line change
`@@ -8,6 +8,7 @@ import (`
`8`	`8`	`"github.com/vmware/govmomi/vapi/tags"`
`9`	`9`	`"github.com/vmware/govmomi/vim25/types"`
`10`	`10`	`corev1 "k8s.io/api/core/v1"`
	`11`	`+ "sigs.k8s.io/controller-runtime/pkg/log"`
`11`	`12`	`)`
`12`	`13`
`13`	`14`	`func (t Provider) getObjectByType(ctx context.Context, tag tags.Tag, objT string) (*types.ManagedObjectReference, error) {`
`@@ -164,7 +165,7 @@ func getCategoryObject(name string) *tags.Category {`
`164`	`165`	`func (t Provider) TagsFromVM(ctx context.Context, vm object.VirtualMachine) (map[string]string, error) {`
`165`	`166`	`tagsAttached, err := t.TagManager.ListAttachedTags(ctx, vm.Reference())`
`166`	`167`	`if err != nil {`
`167`		`- fmt.Printf("Failed to list tags for VM %s: %v\n", vm.Name(), err)`
	`168`	`+ log.FromContext(ctx).Error(err, fmt.Sprintf("failed to list tags for VM %s", vm.Name()))`
`168`	`169`	`}`
`169`	`170`	`return extractTagInfo(ctx, t.TagManager, tagsAttached)`
`170`	`171`