Implement CRD migration

Signed-off-by: Stefan Büringer [email protected]

Author: sbueringer

Makefile

@@ -190,7 +190,7 @@ IMPORT_BOSS_VER := v0.28.1
 IMPORT_BOSS := $(abspath $(TOOLS_BIN_DIR)/$(IMPORT_BOSS_BIN))
 IMPORT_BOSS_PKG := k8s.io/code-generator/cmd/import-boss
 
-CAPI_HACK_TOOLS_VER := 065f159469f356c029c7fdef92cce1d2e4b0ebc6 # Note: this a commit ID of from CAPI main, supposed to be in v1.10
+CAPI_HACK_TOOLS_VER := c9261b079e0bb619a1b23c5a034c660e5660deb7 # Note: this a commit ID of from CAPI main, supposed to be in v1.10
 
 BOSKOSCTL_BIN := boskosctl
 BOSKOSCTL := $(abspath $(TOOLS_BIN_DIR)/$(BOSKOSCTL_BIN))

config/rbac/role.yaml

@@ -55,6 +55,45 @@ rules:
   - services/status
   verbs:
   - get
+- apiGroups:
+  - apiextensions.k8s.io
+  resources:
+  - customresourcedefinitions
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - apiextensions.k8s.io
+  resourceNames:
+  - providerserviceaccounts.vmware.infrastructure.cluster.x-k8s.io
+  - vsphereclusters.vmware.infrastructure.cluster.x-k8s.io
+  - vsphereclustertemplates.vmware.infrastructure.cluster.x-k8s.io
+  - vspheremachines.vmware.infrastructure.cluster.x-k8s.io
+  - vspheremachinetemplates.vmware.infrastructure.cluster.x-k8s.io
+  resources:
+  - customresourcedefinitions
+  - customresourcedefinitions/status
+  verbs:
+  - patch
+  - update
+- apiGroups:
+  - apiextensions.k8s.io
+  resourceNames:
+  - vsphereclusteridentities.infrastructure.cluster.x-k8s.io
+  - vsphereclusters.infrastructure.cluster.x-k8s.io
+  - vsphereclustertemplates.infrastructure.cluster.x-k8s.io
+  - vspheredeploymentzones.infrastructure.cluster.x-k8s.io
+  - vspherefailuredomains.infrastructure.cluster.x-k8s.io
+  - vspheremachines.infrastructure.cluster.x-k8s.io
+  - vspheremachinetemplates.infrastructure.cluster.x-k8s.io
+  - vspherevms.infrastructure.cluster.x-k8s.io
+  resources:
+  - customresourcedefinitions
+  - customresourcedefinitions/status
+  verbs:
+  - patch
+  - update
 - apiGroups:
   - authentication.k8s.io
   resources:
@@ -141,10 +180,13 @@ rules:
 - apiGroups:
   - infrastructure.cluster.x-k8s.io
   resources:
+  - vsphereclustertemplates
   - vspheremachinetemplates
   verbs:
   - get
   - list
+  - patch
+  - update
   - watch
 - apiGroups:
   - ipam.cluster.x-k8s.io
@@ -242,6 +284,8 @@ rules:
   verbs:
   - get
   - list
+  - patch
+  - update
   - watch
 - apiGroups:
   - vmware.infrastructure.cluster.x-k8s.io

controllers/vmware/serviceaccount_controller.go

@@ -52,7 +52,7 @@ import (
 	"sigs.k8s.io/cluster-api-provider-vsphere/pkg/util"
 )
 
-// +kubebuilder:rbac:groups=vmware.infrastructure.cluster.x-k8s.io,resources=providerserviceaccounts,verbs=get;list;watch;
+// +kubebuilder:rbac:groups=vmware.infrastructure.cluster.x-k8s.io,resources=providerserviceaccounts,verbs=get;list;watch;patch;update
 // +kubebuilder:rbac:groups=vmware.infrastructure.cluster.x-k8s.io,resources=providerserviceaccounts/status,verbs=get;update;patch
 // +kubebuilder:rbac:groups="",resources=serviceaccounts,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups="",resources=secrets,verbs=get

main.go

@@ -42,6 +42,7 @@ import (
 	"k8s.io/utils/ptr"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	"sigs.k8s.io/cluster-api/controllers/clustercache"
+	"sigs.k8s.io/cluster-api/controllers/crdmigrator"
 	"sigs.k8s.io/cluster-api/controllers/remote"
 	"sigs.k8s.io/cluster-api/util/apiwarnings"
 	capiflags "sigs.k8s.io/cluster-api/util/flags"
@@ -93,6 +94,7 @@ var (
 	vSphereVMConcurrency              int
 	vSphereClusterIdentityConcurrency int
 	vSphereDeploymentZoneConcurrency  int
+	skipCRDMigrationPhases            []string
 
 	managerOptions = capiflags.ManagerOptions{}
 
@@ -187,6 +189,9 @@ func InitFlags(fs *pflag.FlagSet) {
 	fs.BoolVar(&enableContentionProfiling, "contention-profiling", false,
 		"Enable block profiling.")
 
+	fs.StringArrayVar(&skipCRDMigrationPhases, "skip-crd-migration-phases", []string{},
+		"List of CRD migration phases to skip. Valid values are: StorageVersionMigration, CleanupManagedFields.")
+
 	fs.DurationVar(&syncPeriod, "sync-period", defaultSyncPeriod,
 		"The minimum interval at which watched resources are reconciled (e.g. 15m)")
 
@@ -225,6 +230,14 @@ func InitFlags(fs *pflag.FlagSet) {
 // Add RBAC for the authorized diagnostics endpoint.
 // +kubebuilder:rbac:groups=authentication.k8s.io,resources=tokenreviews,verbs=create
 // +kubebuilder:rbac:groups=authorization.k8s.io,resources=subjectaccessreviews,verbs=create
+// ADD CRD RBAC for CRD Migrator.
+// +kubebuilder:rbac:groups=apiextensions.k8s.io,resources=customresourcedefinitions,verbs=get;list;watch
+// govmomi
+// +kubebuilder:rbac:groups=apiextensions.k8s.io,resources=customresourcedefinitions;customresourcedefinitions/status,verbs=update;patch,resourceNames=vsphereclusters.infrastructure.cluster.x-k8s.io;vsphereclustertemplates.infrastructure.cluster.x-k8s.io;vspheremachines.infrastructure.cluster.x-k8s.io;vspheremachinetemplates.infrastructure.cluster.x-k8s.io;vspherevms.infrastructure.cluster.x-k8s.io;vsphereclusteridentities.infrastructure.cluster.x-k8s.io;vspheredeploymentzones.infrastructure.cluster.x-k8s.io;vspherefailuredomains.infrastructure.cluster.x-k8s.io
+// supervisor
+// +kubebuilder:rbac:groups=apiextensions.k8s.io,resources=customresourcedefinitions;customresourcedefinitions/status,verbs=update;patch,resourceNames=vsphereclusters.vmware.infrastructure.cluster.x-k8s.io;vsphereclustertemplates.vmware.infrastructure.cluster.x-k8s.io;vspheremachines.vmware.infrastructure.cluster.x-k8s.io;vspheremachinetemplates.vmware.infrastructure.cluster.x-k8s.io;providerserviceaccounts.vmware.infrastructure.cluster.x-k8s.io
+// govmomi CRs
+// +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=vspheremachinetemplates;vsphereclustertemplates,verbs=get;list;watch;patch;update
 
 func main() {
 	InitFlags(pflag.CommandLine)
@@ -317,6 +330,43 @@ func main() {
 			setupLog.Info(fmt.Sprintf("CRD for %s not loaded, skipping.", supervisorGVR.String()))
 		}
 
+		// Note: The kubebuilder RBAC markers above has to be kept in sync
+		// with the CRDs that should be migrated by this provider.
+		crdMigratorConfig := map[client.Object]crdmigrator.ByObjectConfig{}
+		if isGovmomiCRDLoaded {
+			crdMigratorConfig[&infrav1.VSphereCluster{}] = crdmigrator.ByObjectConfig{UseCache: true}
+			crdMigratorConfig[&infrav1.VSphereClusterTemplate{}] = crdmigrator.ByObjectConfig{UseCache: false}
+			crdMigratorConfig[&infrav1.VSphereMachine{}] = crdmigrator.ByObjectConfig{UseCache: true}
+			crdMigratorConfig[&infrav1.VSphereMachineTemplate{}] = crdmigrator.ByObjectConfig{UseCache: true}
+			crdMigratorConfig[&infrav1.VSphereVM{}] = crdmigrator.ByObjectConfig{UseCache: true}
+			crdMigratorConfig[&infrav1.VSphereClusterIdentity{}] = crdmigrator.ByObjectConfig{UseCache: true}
+			crdMigratorConfig[&infrav1.VSphereDeploymentZone{}] = crdmigrator.ByObjectConfig{UseCache: true}
+			crdMigratorConfig[&infrav1.VSphereFailureDomain{}] = crdmigrator.ByObjectConfig{UseCache: true}
+		}
+		if isSupervisorCRDLoaded {
+			crdMigratorConfig[&vmwarev1.VSphereCluster{}] = crdmigrator.ByObjectConfig{UseCache: true}
+			crdMigratorConfig[&vmwarev1.VSphereClusterTemplate{}] = crdmigrator.ByObjectConfig{UseCache: false}
+			crdMigratorConfig[&vmwarev1.VSphereMachine{}] = crdmigrator.ByObjectConfig{UseCache: true}
+			crdMigratorConfig[&vmwarev1.VSphereMachineTemplate{}] = crdmigrator.ByObjectConfig{UseCache: true}
+			crdMigratorConfig[&vmwarev1.ProviderServiceAccount{}] = crdmigrator.ByObjectConfig{UseCache: true}
+		}
+
+		crdMigratorSkipPhases := []crdmigrator.Phase{}
+		for _, p := range skipCRDMigrationPhases {
+			crdMigratorSkipPhases = append(crdMigratorSkipPhases, crdmigrator.Phase(p))
+		}
+		if err := (&crdmigrator.CRDMigrator{
+			Client:                 mgr.GetClient(),
+			APIReader:              mgr.GetAPIReader(),
+			SkipCRDMigrationPhases: crdMigratorSkipPhases,
+			Config:                 crdMigratorConfig,
+			// The CRDMigrator is run with only concurrency 1 to ensure we don't overwhelm the apiserver by patching a
+			// lot of CRs concurrently.
+		}).SetupWithManager(ctx, mgr, concurrency(1)); err != nil {
+			setupLog.Error(err, "Unable to create controller", "controller", "CRDMigrator")
+			os.Exit(1)
+		}
+
 		return nil
 	}
 

pkg/manager/manager.go

@@ -26,6 +26,7 @@ import (
 	vmoprv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2"
 	ncpv1 "github.com/vmware-tanzu/vm-operator/external/ncp/api/v1alpha1"
 	"gopkg.in/fsnotify.v1"
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
@@ -54,6 +55,7 @@ func New(ctx context.Context, opts Options) (Manager, error) {
 	// Ensure the default options are set.
 	opts.defaults()
 
+	_ = apiextensionsv1.AddToScheme(opts.Scheme)
 	_ = clientgoscheme.AddToScheme(opts.Scheme)
 	_ = clusterv1.AddToScheme(opts.Scheme)
 	_ = infrav1alpha3.AddToScheme(opts.Scheme)

test/e2e/clusterctl_upgrade_test.go

@@ -19,9 +19,13 @@ package e2e
 import (
 	"context"
 	"fmt"
+	"strings"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
+	clusterctlcluster "sigs.k8s.io/cluster-api/cmd/clusterctl/client/cluster"
 	capi_e2e "sigs.k8s.io/cluster-api/test/e2e"
 	"sigs.k8s.io/cluster-api/test/framework"
 	"sigs.k8s.io/cluster-api/test/framework/clusterctl"
@@ -67,6 +71,15 @@ var _ = Describe("When testing clusterctl upgrades using ClusterClass (CAPV 1.12
 				InitWithInfrastructureProviders:   []string{fmt.Sprintf(providerVSpherePrefix, capvStableRelease)},
 				InitWithRuntimeExtensionProviders: testSpecificSettingsGetter().RuntimeExtensionProviders,
 				InitWithIPAMProviders:             []string{},
+				Upgrades: []capi_e2e.ClusterctlUpgradeSpecInputUpgrade{
+					{ // Upgrade to latest v1beta1.
+						Contract: clusterv1.GroupVersion.Version,
+						PostUpgrade: func(proxy framework.ClusterProxy, namespace, clusterName string) {
+							framework.ValidateCRDMigration(ctx, proxy, namespace, clusterName,
+								crdShouldBeMigrated, clusterctlcluster.FilterClusterObjectsWithNameFilter(clusterName))
+						},
+					},
+				},
 				// InitWithKubernetesVersion should be the highest kubernetes version supported by the init Cluster API version.
 				// This is to guarantee that both, the old and new CAPI version, support the defined version.
 				// Ensure all Kubernetes versions used here are covered in patch-vsphere-template.yaml
@@ -112,6 +125,15 @@ var _ = Describe("When testing clusterctl upgrades using ClusterClass (CAPV 1.12
 				InitWithInfrastructureProviders:   []string{fmt.Sprintf(providerVSpherePrefix, capvStableRelease)},
 				InitWithRuntimeExtensionProviders: testSpecificSettingsGetter().RuntimeExtensionProviders,
 				InitWithIPAMProviders:             []string{},
+				Upgrades: []capi_e2e.ClusterctlUpgradeSpecInputUpgrade{
+					{ // Upgrade to latest v1beta1.
+						Contract: clusterv1.GroupVersion.Version,
+						PostUpgrade: func(proxy framework.ClusterProxy, namespace, clusterName string) {
+							framework.ValidateCRDMigration(ctx, proxy, namespace, clusterName,
+								crdShouldBeMigrated, clusterctlcluster.FilterClusterObjectsWithNameFilter(clusterName))
+						},
+					},
+				},
 				// InitWithKubernetesVersion should be the highest kubernetes version supported by the init Cluster API version.
 				// This is to guarantee that both, the old and new CAPI version, support the defined version.
 				// Ensure all Kubernetes versions used here are covered in patch-vsphere-template.yaml
@@ -157,6 +179,15 @@ var _ = Describe("When testing clusterctl upgrades using ClusterClass (CAPV 1.11
 				InitWithInfrastructureProviders:   []string{fmt.Sprintf(providerVSpherePrefix, capvStableRelease)},
 				InitWithRuntimeExtensionProviders: testSpecificSettingsGetter().RuntimeExtensionProviders,
 				InitWithIPAMProviders:             []string{},
+				Upgrades: []capi_e2e.ClusterctlUpgradeSpecInputUpgrade{
+					{ // Upgrade to latest v1beta1.
+						Contract: clusterv1.GroupVersion.Version,
+						PostUpgrade: func(proxy framework.ClusterProxy, namespace, clusterName string) {
+							framework.ValidateCRDMigration(ctx, proxy, namespace, clusterName,
+								crdShouldBeMigrated, clusterctlcluster.FilterClusterObjectsWithNameFilter(clusterName))
+						},
+					},
+				},
 				// InitWithKubernetesVersion should be the highest kubernetes version supported by the init Cluster API version.
 				// This is to guarantee that both, the old and new CAPI version, support the defined version.
 				// Ensure all Kubernetes versions used here are covered in patch-vsphere-template.yaml
@@ -202,6 +233,15 @@ var _ = Describe("When testing clusterctl upgrades using ClusterClass (CAPV 1.10
 				InitWithInfrastructureProviders:   []string{fmt.Sprintf(providerVSpherePrefix, capvStableRelease)},
 				InitWithRuntimeExtensionProviders: testSpecificSettingsGetter().RuntimeExtensionProviders,
 				InitWithIPAMProviders:             []string{},
+				Upgrades: []capi_e2e.ClusterctlUpgradeSpecInputUpgrade{
+					{ // Upgrade to latest v1beta1.
+						Contract: clusterv1.GroupVersion.Version,
+						PostUpgrade: func(proxy framework.ClusterProxy, namespace, clusterName string) {
+							framework.ValidateCRDMigration(ctx, proxy, namespace, clusterName,
+								crdShouldBeMigrated, clusterctlcluster.FilterClusterObjectsWithNameFilter(clusterName))
+						},
+					},
+				},
 				// InitWithKubernetesVersion should be the highest kubernetes version supported by the init Cluster API version.
 				// This is to guarantee that both, the old and new CAPI version, support the defined version.
 				// Ensure all Kubernetes versions used here are covered in patch-vsphere-template.yaml
@@ -233,3 +273,8 @@ func kindManagementClusterNewClusterProxyFunc(name string, kubeconfigPath string
 	}
 	return framework.NewClusterProxy(name, kubeconfigPath, initScheme())
 }
+
+func crdShouldBeMigrated(crd apiextensionsv1.CustomResourceDefinition) bool {
+	return strings.HasSuffix(crd.Name, ".infrastructure.cluster.x-k8s.io") && // govmomi & supervisor
+		!strings.HasSuffix(crd.Name, ".vcsim.infrastructure.cluster.x-k8s.io") // !vcsim
+}

test/e2e/e2e_suite_test.go

@@ -318,7 +318,9 @@ var _ = SynchronizedAfterSuite(func() {
 
 		case VCSimTestTarget:
 			// Cleanup the vcsim address manager
-			Expect(vcsimAddressManager.Teardown(ctx)).To(Succeed())
+			if vcsimAddressManager != nil {
+				Expect(vcsimAddressManager.Teardown(ctx)).To(Succeed())
+			}
 
 			// cleanup the vcsim server
 			Expect(vspherevcsim.Delete(ctx, bootstrapClusterProxy.GetClient(), skipCleanup)).To(Succeed())