[WIP] experimental support for vm-operator-api

.ci/check

@@ -23,7 +23,7 @@ go install -mod=vendor golang.org/x/lint/golint
 export GOFLAGS=-mod=vendor
 
 ###############################################################################
-PACKAGES="$(go list -e ./... | grep -vE '/vendor/|/internal/flags')"
+PACKAGES="$(go list -e ./... | grep -vE '/vendor/|/internal/vmomi/flags')"
 PACKAGES_DIRS="$(echo ${PACKAGES} | sed "s|github.com/gardener/machine-controller-manager-provider-vsphere|.|g")"
 
 # Execute static code checks.

go.mod

@@ -4,27 +4,28 @@ go 1.15
 
 require (
 	github.com/gardener/machine-controller-manager v0.39.0
-	github.com/onsi/ginkgo v1.15.2
-	github.com/onsi/gomega v1.11.0
+	github.com/onsi/ginkgo v1.16.4
+	github.com/onsi/gomega v1.13.0
 	github.com/pkg/errors v0.9.1
-	github.com/prometheus/client_golang v1.5.1 // indirect
 	github.com/spf13/pflag v1.0.5
-	github.com/stretchr/testify v1.5.1 // indirect
+	github.com/vmware-tanzu/vm-operator-api v0.1.4-0.20210722184632-99fee0b6197e
 	github.com/vmware/govmomi v0.22.1
-	golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f
-	golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb
-	k8s.io/api v0.16.8
-	k8s.io/component-base v0.16.8
-	k8s.io/klog v1.0.0
+	golang.org/x/lint v0.0.0-20200302205851-738671d3881b
+	golang.org/x/net v0.0.0-20210428140749-89ef3d95e781
+	k8s.io/api v0.17.11
+	k8s.io/apimachinery v0.17.11
+	k8s.io/client-go v0.17.11
+	k8s.io/component-base v0.17.11
+	k8s.io/klog/v2 v2.9.0
+	sigs.k8s.io/controller-runtime v0.5.10
 	sigs.k8s.io/yaml v1.2.0
 )
 
 replace (
-	github.com/prometheus/client_golang => github.com/prometheus/client_golang v0.9.2
-	k8s.io/api => k8s.io/api v0.16.8 // v0.16.8
-	k8s.io/apimachinery => k8s.io/apimachinery v0.16.8 // v0.16.8
-	k8s.io/apiserver => k8s.io/apiserver v0.16.8 // v0.16.8
-	k8s.io/client-go => k8s.io/client-go v0.16.8 // v0.16.8
-	k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.16.8 // v0.16.8
-	k8s.io/kube-openapi => k8s.io/kube-openapi v0.0.0-20190816220812-743ec37842bf
+	github.com/prometheus/client_golang => github.com/prometheus/client_golang v0.9.2 // keep this value in sync with sigs.k8s.io/controller-runtime	k8s.io/api => k8s.io/api v0.17.11
+	k8s.io/apimachinery => k8s.io/apimachinery v0.17.11
+	k8s.io/apiserver => k8s.io/apiserver v0.17.11
+	k8s.io/client-go => k8s.io/client-go v0.17.11
+	k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.17.11
+	k8s.io/kube-openapi => github.com/gardener/kube-openapi v0.0.0-20200807191151-9232ec702af2
 )

pkg/vsphere/apis/provider_spec.go

@@ -23,11 +23,30 @@ const (
 	TagMCMClusterName = "mcm.gardener.cloud/cluster"
 	// TagMCMRole is the tag key for tagging a VM with its role (e.g 'node')
 	TagMCMRole = "mcm.gardener.cloud/role"
+
+	// LabelMCMVSphere is the tag key for labeling a VM as a managed machine
+	LabelMCMVSphere = "mcm.gardener.cloud/machine"
+
+	// VSphereKubeconfig is the key for the kubeconfig in the credentials secret
+	VSphereKubeconfig = "vsphereKubeconfig"
 )
 
-// VsphereProviderSpec contains the fields of
-// provider spec that the plugin expects
+// VsphereProviderSpec is an interface to hide the concrete spec version
 type VsphereProviderSpec struct {
+	V1 *VsphereProviderSpec1 `json:"v1,omitempty"`
+	V2 *VsphereProviderSpec2 `json:"v2,omitempty"`
+
+	// SSHKeys is an optional array of ssh public keys to deploy to VM (may already be included in UserData)
+	// +optional
+	SSHKeys []string `json:"sshKeys,omitempty"`
+	// Tags to be placed on the VM
+	// +optional
+	Tags map[string]string `json:"tags,omitempty"`
+}
+
+// VsphereProviderSpec1 contains the fields of
+// provider spec that the plugin expects
+type VsphereProviderSpec1 struct {
 	// Region is the vSphere region
 	Region string `json:"region"`
 
@@ -89,15 +108,56 @@ type VsphereProviderSpec struct {
 	// Customization is an experimental option to add a CustomizationSpec
 	// +optional
 	Customization string `json:"customization,omitempty"`
+}
 
-	// SSHKeys is an optional array of ssh public keys to deploy to VM (may already be included in UserData)
+// SpecVersion returns spec version
+func (s *VsphereProviderSpec1) SpecVersion() int { return 1 }
+
+// VsphereProviderSpec2 contains the fields of
+// provider spec that the plugin expects
+type VsphereProviderSpec2 struct {
+	// Namespace is the vSphere workload namespace
+	Namespace string `json:"namespace"`
+
+	// ImageName describes the name of a VirtualMachineImage that is to be used as the base Operating System image of
+	// the desired VirtualMachine instances.  The VirtualMachineImage resources can be introspected to discover identifying
+	// attributes that may help users to identify the desired image to use.
+	ImageName string `json:"imageName"`
+
+	// ClassName describes the name of a VirtualMachineClass that is to be used as the overlaid resource configuration
+	// of VirtualMachine.  A VirtualMachineClass is used to further customize the attributes of the VirtualMachine
+	// instance.  See VirtualMachineClass for more description.
+	ClassName string `json:"className"`
+
+	// NetworkName is the name of the network for the virtualmachines.vmoperator.vmware.com resource
+	NetworkName string `json:"networkName"`
+	// NetworkType is the type of the network for the virtualmachines.vmoperator.vmware.com resource
+	NetworkType string `json:"networkType"`
+
+	// StorageClass describes the name of a StorageClass that should be used to configure storage-related attributes of the VirtualMachine
+	// instance.
 	// +optional
-	SSHKeys []string `json:"sshKeys,omitempty"`
-	// Tags to be placed on the VM
+	StorageClass *string `json:"storageClass,omitempty"`
+
+	// ResourcePolicyName describes the name of a VirtualMachineSetResourcePolicy to be used when creating the
+	// VirtualMachine instance.
 	// +optional
-	Tags map[string]string `json:"tags,omitempty"`
+	ResourcePolicyName *string `json:"resourcePolicyName,omitempty"`
+
+	// SystemDisk specifies the system disk
+	// +optional
+	SystemDisk *VSphereSystemDisk `json:"systemDisk,omitempty"`
+
+	// TODO
+	// ExtraConfig allows to specify additional VM options.
+	// e.g. sched.swap.vmxSwapEnabled=false to disable the VMX process swap file
+	// +optional
+	//ExtraConfig map[string]string `json:"extraConfig,omitempty"`
 }
 
+// SpecVersion returns spec version
+func (s *VsphereProviderSpec2) SpecVersion() int { return 2 }
+
 // VSphereSystemDisk specifies system disk of a machine
 type VSphereSystemDisk struct {
 	// Size is disk size in GB

pkg/vsphere/apis/tags/tagutils.go

@@ -93,3 +93,27 @@ func (t *RelevantTags) Matches(tags map[string]string) bool {
 	}
 	return matchedCluster && matchedRole
 }
+
+// GetLabels returns label for relevant tags
+func (t *RelevantTags) GetLabels() map[string]string {
+	return map[string]string{
+		api.TagMCMClusterName: t.clusterName,
+		api.TagMCMRole:        t.nodeRole,
+	}
+}
+
+// NonRelevant returns a copy of the input map without the relevant tags
+func (t *RelevantTags) NonRelevant(tags map[string]string) map[string]string {
+	result := map[string]string{}
+	for key, value := range tags {
+		switch key {
+		case t.clusterNameKey:
+		case t.nodeRoleKey:
+		case api.TagMCMClusterName:
+		case api.TagMCMRole:
+		default:
+			result[key] = value
+		}
+	}
+	return result
+}

pkg/vsphere/apis/validation/validation.go

@@ -25,20 +25,21 @@ import (
 	corev1 "k8s.io/api/core/v1"
 )
 
-// ValidateVsphereProviderSpec validates Vsphere provider spec
-func ValidateVsphereProviderSpec(spec *api.VsphereProviderSpec, secrets *corev1.Secret) []error {
+// ValidateVsphereProviderSpec1 validates Vsphere provider spec
+func ValidateVsphereProviderSpec1(spec *api.VsphereProviderSpec, secrets *corev1.Secret) []error {
 	var allErrs []error
 
-	if "" == spec.Datastore && "" == spec.DatastoreCluster {
+	v1 := spec.V1
+	if "" == v1.Datastore && "" == v1.DatastoreCluster {
 		allErrs = append(allErrs, fmt.Errorf("either datastoreCluster or datastore field is required"))
 	}
-	if "" == spec.TemplateVM {
+	if "" == v1.TemplateVM {
 		allErrs = append(allErrs, fmt.Errorf("templateVM is a required field"))
 	}
-	if "" == spec.ComputeCluster && "" == spec.ResourcePool && "" == spec.HostSystem {
+	if "" == v1.ComputeCluster && "" == v1.ResourcePool && "" == v1.HostSystem {
 		allErrs = append(allErrs, fmt.Errorf("either computeCluster or resourcePool or hostSystem field is required"))
 	}
-	if "" == spec.Network {
+	if "" == v1.Network {
 		allErrs = append(allErrs, fmt.Errorf("network is a required field"))
 	}
 
@@ -76,3 +77,51 @@ func validateSecrets(secret *corev1.Secret) []error {
 
 	return allErrs
 }
+
+// ValidateVsphereProviderSpec2 validates Vsphere provider spec2
+func ValidateVsphereProviderSpec2(spec *api.VsphereProviderSpec, secrets *corev1.Secret) []error {
+	var allErrs []error
+
+	v2 := spec.V2
+	if "" == v2.Namespace {
+		allErrs = append(allErrs, fmt.Errorf("namespace is a required field"))
+	}
+	if "" == v2.ImageName {
+		allErrs = append(allErrs, fmt.Errorf("imageName is a required field"))
+	}
+	if "" == v2.NetworkType {
+		allErrs = append(allErrs, fmt.Errorf("networkType is a required field"))
+	}
+	if "" == v2.NetworkName {
+		allErrs = append(allErrs, fmt.Errorf("networkName is a required field"))
+	}
+	if "" == v2.ClassName {
+		allErrs = append(allErrs, fmt.Errorf("className is a required field"))
+	}
+
+	allErrs = append(allErrs, validateSecrets2(secrets)...)
+	_, tagErrs := tags.NewRelevantTags(spec.Tags)
+	allErrs = append(allErrs, tagErrs...)
+
+	return allErrs
+}
+
+func validateSecrets2(secret *corev1.Secret) []error {
+	var allErrs []error
+
+	if secret == nil {
+		allErrs = append(allErrs, fmt.Errorf("Secret object that has been passed by the MCM is nil"))
+	} else {
+		_, kubeconfigExists := secret.Data[api.VSphereKubeconfig]
+		_, userDataExists := secret.Data["userData"]
+
+		if !kubeconfigExists {
+			allErrs = append(allErrs, fmt.Errorf("Secret %s is required field", api.VSphereKubeconfig))
+		}
+		if !userDataExists {
+			allErrs = append(allErrs, fmt.Errorf("Secret userData is required field"))
+		}
+	}
+
+	return allErrs
+}

pkg/vsphere/errors/errors.go

@@ -23,10 +23,15 @@ import "fmt"
 type MachineNotFoundError struct {
 	// Name is the machine name
 	Name string
-	// MachineID is the machine uuid
+	// MachineID is the machine uuid (only used for spec1)
 	MachineID string
+	// Namespace is namespace (only used for spec2)
+	Namespace string
 }
 
 func (e *MachineNotFoundError) Error() string {
+	if e.Namespace != "" {
+		return fmt.Sprintf("machine %s/%s not found", e.Namespace, e.Name)
+	}
 	return fmt.Sprintf("machine name=%s, uuid=%s not found", e.Name, e.MachineID)
 }

pkg/vsphere/integration/integration_test.go

@@ -60,10 +60,16 @@ func TestPluginSPIImpl(t *testing.T) {
 		return
 	}
 
-	spi := &internal.PluginSPIImpl{}
+	providerSpec := cfg.ProviderSpec
+	if providerSpec.V1 == nil && providerSpec.V2 == nil {
+		t.Errorf("neither field 'providerSpec.v1' nor 'providerSpec.v2' set in integrationConfig from %s", configPath)
+		return
+	}
+
+	spi := &internal.PluginSPISwitch{}
 	ctx := context.TODO()
 
-	providerID, err := spi.GetMachineStatus(ctx, cfg.MachineName, "", cfg.ProviderSpec, cfg.Secrets)
+	providerID, err := spi.GetMachineStatus(ctx, cfg.MachineName, "", providerSpec, cfg.Secrets)
 	if err == nil {
 		t.Errorf("Machine name %s already existing", cfg.MachineName)
 		return
@@ -76,7 +82,7 @@ func TestPluginSPIImpl(t *testing.T) {
 		return
 	}
 
-	providerID, err = spi.DeleteMachine(ctx, cfg.MachineName, providerID, cfg.ProviderSpec, cfg.Secrets)
+	providerID, err = spi.DeleteMachine(ctx, cfg.MachineName, providerID, providerSpec, cfg.Secrets)
 	switch err.(type) {
 	case *errors.MachineNotFoundError:
 		// expected
@@ -85,13 +91,13 @@ func TestPluginSPIImpl(t *testing.T) {
 		return
 	}
 
-	providerID, err = spi.CreateMachine(ctx, cfg.MachineName, cfg.ProviderSpec, cfg.Secrets)
+	providerID, err = spi.CreateMachine(ctx, cfg.MachineName, providerSpec, cfg.Secrets)
 	if err != nil {
 		t.Errorf("CreateMachine failed with %s", err)
 		return
 	}
 
-	providerID2, err := spi.GetMachineStatus(ctx, cfg.MachineName, "", cfg.ProviderSpec, cfg.Secrets)
+	providerID2, err := spi.GetMachineStatus(ctx, cfg.MachineName, "", providerSpec, cfg.Secrets)
 	if err != nil {
 		t.Errorf("GetMachineStatus by machine name failed with %s", err)
 		return
@@ -100,7 +106,7 @@ func TestPluginSPIImpl(t *testing.T) {
 		t.Errorf("ProviderID mismatch %s != %s", providerID, providerID2)
 	}
 
-	providerID2, err = spi.GetMachineStatus(ctx, cfg.MachineName, providerID, cfg.ProviderSpec, cfg.Secrets)
+	providerID2, err = spi.GetMachineStatus(ctx, cfg.MachineName, providerID, providerSpec, cfg.Secrets)
 	if err != nil {
 		t.Errorf("GetMachineStatus by providerID failed with %s", err)
 		return
@@ -109,7 +115,7 @@ func TestPluginSPIImpl(t *testing.T) {
 		t.Errorf("ProviderID mismatch %s != %s", providerID, providerID2)
 	}
 
-	providerIDList, err := spi.ListMachines(ctx, cfg.ProviderSpec, cfg.Secrets)
+	providerIDList, err := spi.ListMachines(ctx, providerSpec, cfg.Secrets)
 	if err != nil {
 		t.Errorf("ListMachines failed with %s", err)
 	}
@@ -127,15 +133,15 @@ func TestPluginSPIImpl(t *testing.T) {
 		t.Errorf("Created machine with ID %s not found", providerID)
 	}
 
-	providerID2, err = spi.ShutDownMachine(ctx, cfg.MachineName, providerID, cfg.ProviderSpec, cfg.Secrets)
+	providerID2, err = spi.ShutDownMachine(ctx, cfg.MachineName, providerID, providerSpec, cfg.Secrets)
 	if err != nil {
 		t.Errorf("ShutDownMachine failed with %s", err)
 	}
 	if providerID != providerID2 {
 		t.Errorf("ProviderID mismatch %s != %s", providerID, providerID2)
 	}
 
-	providerID2, err = spi.DeleteMachine(ctx, cfg.MachineName, providerID, cfg.ProviderSpec, cfg.Secrets)
+	providerID2, err = spi.DeleteMachine(ctx, cfg.MachineName, providerID, providerSpec, cfg.Secrets)
 	if err != nil {
 		t.Errorf("DeleteMachine failed with %s", err)
 	}