Merge pull request #7288 from fabriziopandini/support-ipam-and-runtime-extension-providers

✨ Support IPAM and runtime extension providers

Author: k8s-ci-robot

Makefile

@@ -612,8 +612,8 @@ docker-capd-build-all: $(addprefix docker-capd-build-,$(ALL_ARCH)) ## Build capd
 .PHONY: docker-build-test-extension
 docker-build-test-extension: ## Build the docker image for core controller manager
 	DOCKER_BUILDKIT=1 docker build --build-arg builder_image=$(GO_CONTAINER_IMAGE) --build-arg goproxy=$(GOPROXY) --build-arg ARCH=$(ARCH) --build-arg ldflags="$(LDFLAGS)" . -t $(TEST_EXTENSION_IMG)-$(ARCH):$(TAG) --file ./test/extension/Dockerfile
-	$(MAKE) set-manifest-image MANIFEST_IMG=$(TEST_EXTENSION_IMG)-$(ARCH) MANIFEST_TAG=$(TAG) TARGET_RESOURCE="./test/extension/config/default/extension_image_patch.yaml"
-	$(MAKE) set-manifest-pull-policy TARGET_RESOURCE="./test/extension/config/default/extension_pull_policy.yaml"
+	$(MAKE) set-manifest-image MANIFEST_IMG=$(TEST_EXTENSION_IMG)-$(ARCH) MANIFEST_TAG=$(TAG) TARGET_RESOURCE="./test/extension/config/default/manager_image_patch.yaml"
+	$(MAKE) set-manifest-pull-policy TARGET_RESOURCE="./test/extension/config/default/manager_pull_policy.yaml"
 
 .PHONY: e2e-framework
 e2e-framework: ## Builds the CAPI e2e framework
@@ -710,6 +710,7 @@ tilt-up: kind-cluster ## Start tilt and build kind cluster if needed.
 docker-build-e2e: ## Rebuild all Cluster API provider images to be used in the e2e tests
 	$(MAKE) docker-build REGISTRY=gcr.io/k8s-staging-cluster-api PULL_POLICY=IfNotPresent
 	$(MAKE) docker-capd-build REGISTRY=gcr.io/k8s-staging-cluster-api PULL_POLICY=IfNotPresent
+	$(MAKE) docker-build-test-extension REGISTRY=gcr.io/k8s-staging-cluster-api PULL_POLICY=IfNotPresent
 
 ## --------------------------------------
 ## Release

Tiltfile

@@ -104,30 +104,30 @@ providers = {
             "internal",
             "third_party",
         ],
+        "label": "CAPD",
+        # Add kubectl to the docker image, CAPD manager requires it.
         "additional_docker_helper_commands": "RUN curl -LO https://dl.k8s.io/release/{KUBE}/bin/linux/{ARCH}/kubectl && chmod +x ./kubectl && mv ./kubectl /usr/bin/kubectl".format(
             ARCH = os_arch,
             KUBE = kubernetes_version,
         ),
         "additional_docker_build_commands": """
 COPY --from=tilt-helper /usr/bin/kubectl /usr/bin/kubectl
 """,
-        "label": "CAPD",
     },
-}
-
-# Create a data structure to hold information about addons.
-addons = {
     "test-extension": {
-        "context": "./test/extension",
+        "context": "test/extension",
         "image": "gcr.io/k8s-staging-cluster-api/test-extension",
-        "container_name": "extension",
-        "live_reload_deps": ["main.go", "handlers"],
+        "live_reload_deps": [
+            "main.go",
+            "handlers",
+        ],
         "label": "test-extension",
-        "resource_deps": ["capi_controller"],
+        # Add the ExtensionConfig for this Runtime extension; given that the ExtensionConfig can be installed only when capi_controller
+        # are up and running, it is required to set a resource_deps to ensure proper install order.
         "additional_resources": [
             "config/tilt/extensionconfig.yaml",
-            "config/tilt/hookresponses-configmap.yaml",
         ],
+        "resource_deps": ["capi_controller"],
     },
 }
 
@@ -165,27 +165,6 @@ def load_provider_tiltfiles():
                 provider_config["go_main"] = "main.go"
             providers[provider_name] = provider_config
 
-# load_addon_tiltfiles looks for tilt-addon.[yaml|json] files in the repositories listed in "addon_repos" in tilt settings and loads their config.
-def load_addon_tiltfiles():
-    addon_repos = settings.get("addon_repos", [])
-    for repo in addon_repos:
-        file = repo + "/tilt-addon.yaml" if os.path.exists(repo + "/tilt-addon.yaml") else repo + "/tilt-addon.json"
-        if not os.path.exists(file):
-            fail("Failed to load provider. No tilt-addon.{yaml|json} file found in " + repo)
-        addon_details = read_yaml(file, default = {})
-        if type(addon_details) != type([]):
-            addon_details = [addon_details]
-        for item in addon_details:
-            addon_name = item["name"]
-            addon_config = item["config"]
-            if "context" in addon_config:
-                addon_config["context"] = repo + "/" + addon_config["context"]
-            else:
-                addon_config["context"] = repo
-            if "go_main" not in addon_config:
-                addon_config["go_main"] = "main.go"
-            addons[addon_name] = addon_config
-
 tilt_helper_dockerfile_header = """
 # Tilt image
 FROM golang:1.19.0 as tilt-helper
@@ -312,7 +291,6 @@ def get_port_forwards(debug):
 # 2. Configures a docker build for the provider, with live updating of the manager binary
 # 3. Runs kustomize for the provider's config/default and applies it
 def enable_provider(name, debug):
-    deployment_kind = "provider"
     p = providers.get(name)
     label = p.get("label")
 
@@ -336,77 +314,26 @@ def enable_provider(name, debug):
         port_forwards = port_forwards,
     )
 
+    additional_objs = []
+    p_resources = p.get("additional_resources", [])
+    for resource in p_resources:
+        k8s_yaml(p.get("context") + "/" + resource)
+        additional_objs = additional_objs + decode_yaml_stream(read_file(p.get("context") + "/" + resource))
+
     if p.get("kustomize_config", True):
-        yaml = read_file("./.tiltbuild/yaml/{}.{}.yaml".format(name, deployment_kind))
+        yaml = read_file("./.tiltbuild/yaml/{}.provider.yaml".format(name))
         k8s_yaml(yaml)
         objs = decode_yaml_stream(yaml)
         k8s_resource(
             workload = find_object_name(objs, "Deployment"),
-            objects = [find_object_qualified_name(objs, "Provider")],
+            objects = [find_object_qualified_name(objs, "Provider")] + find_all_objects_names(additional_objs),
             new_name = label.lower() + "_controller",
             labels = [label, "ALL.controllers"],
             port_forwards = port_forwards,
             links = links,
-            resource_deps = ["provider_crd"],
-        )
-
-# Configures an addon by doing the following:
-#
-# 1. Enables a local_resource go build of the addon's manager binary
-# 2. Configures a docker build for the addon, with live updating of the binary
-# 3. Runs kustomize for the addons's config/default and applies it
-def enable_addon(name, debug):
-    addon = addons.get(name)
-    deployment_kind = "addon"
-    label = addon.get("label")
-    port_forwards, links = get_port_forwards(debug)
-
-    build_go_binary(
-        context = addon.get("context"),
-        reload_deps = addon.get("live_reload_deps"),
-        debug = debug,
-        go_main = addon.get("go_main", "main.go"),
-        binary_name = "extension",
-        label = label,
-    )
-
-    build_docker_image(
-        image = addon.get("image"),
-        context = addon.get("context"),
-        binary_name = "extension",
-        additional_docker_helper_commands = addon.get("additional_docker_helper_commands", ""),
-        additional_docker_build_commands = addon.get("additional_docker_build_commands", ""),
-        port_forwards = port_forwards,
-    )
-
-    additional_objs = []
-    addon_resources = addon.get("additional_resources", [])
-    for resource in addon_resources:
-        k8s_yaml(addon.get("context") + "/" + resource)
-        additional_objs = additional_objs + decode_yaml_stream(read_file(addon.get("context") + "/" + resource))
-
-    if addon.get("kustomize_config", True):
-        yaml = read_file("./.tiltbuild/yaml/{}.{}.yaml".format(name, deployment_kind))
-        objs = decode_yaml_stream(yaml)
-        k8s_yaml(yaml)
-        k8s_resource(
-            workload = find_object_name(objs, "Deployment"),
-            new_name = label.lower() + "_addon",
-            labels = [label, "ALL.addons"],
-            port_forwards = port_forwards,
-            links = links,
-            objects = find_all_objects_names(additional_objs),
-            resource_deps = addon.get("resource_deps", {}),
+            resource_deps = ["provider_crd"] + p.get("resource_deps", []),
         )
 
-def enable_addons():
-    for name in get_addons():
-        enable_addon(name, settings.get("debug").get(name, {}))
-
-def get_addons():
-    user_enable_addons = settings.get("enable_addons", [])
-    return {k: "" for k in user_enable_addons}.keys()
-
 def find_object_name(objs, kind):
     for o in objs:
         if o["kind"] == kind:
@@ -634,8 +561,6 @@ include_user_tilt_files()
 
 load_provider_tiltfiles()
 
-load_addon_tiltfiles()
-
 prepare_all()
 
 deploy_provider_crds()
@@ -644,6 +569,4 @@ deploy_observability()
 
 enable_providers()
 
-enable_addons()
-
 cluster_templates()

cmd/clusterctl/api/v1alpha3/labels.go

@@ -50,6 +50,10 @@ func ManifestLabel(name string, providerType ProviderType) string {
 		return fmt.Sprintf("control-plane-%s", name)
 	case InfrastructureProviderType:
 		return fmt.Sprintf("infrastructure-%s", name)
+	case IPAMProviderType:
+		return fmt.Sprintf("ipam-%s", name)
+	case RuntimeExtensionProviderType:
+		return fmt.Sprintf("runtime-extension-%s", name)
 	default:
 		return name
 	}

cmd/clusterctl/api/v1alpha3/provider_type.go

@@ -92,7 +92,9 @@ func (p *Provider) GetProviderType() ProviderType {
 		CoreProviderType,
 		BootstrapProviderType,
 		InfrastructureProviderType,
-		ControlPlaneProviderType:
+		ControlPlaneProviderType,
+		IPAMProviderType,
+		RuntimeExtensionProviderType:
 		return t
 	default:
 		return ProviderTypeUnknown
@@ -118,6 +120,14 @@ const (
 	// control-plane capabilities.
 	ControlPlaneProviderType = ProviderType("ControlPlaneProvider")
 
+	// IPAMProviderType is the type associated with codebases that provide
+	// IPAM capabilities.
+	IPAMProviderType = ProviderType("IPAMProvider")
+
+	// RuntimeExtensionProviderType is the type associated with codebases that provide
+	// runtime extensions.
+	RuntimeExtensionProviderType = ProviderType("RuntimeExtensionProvider")
+
 	// ProviderTypeUnknown is used when the type is unknown.
 	ProviderTypeUnknown = ProviderType("")
 )
@@ -133,8 +143,12 @@ func (p ProviderType) Order() int {
 		return 2
 	case InfrastructureProviderType:
 		return 3
-	default:
+	case IPAMProviderType:
 		return 4
+	case RuntimeExtensionProviderType:
+		return 5
+	default:
+		return 99
 	}
 }
 

cmd/clusterctl/client/config/providers_client.go

@@ -361,14 +361,18 @@ func validateProvider(r Provider) error {
 	case clusterctlv1.CoreProviderType,
 		clusterctlv1.BootstrapProviderType,
 		clusterctlv1.InfrastructureProviderType,
-		clusterctlv1.ControlPlaneProviderType:
+		clusterctlv1.ControlPlaneProviderType,
+		clusterctlv1.IPAMProviderType,
+		clusterctlv1.RuntimeExtensionProviderType:
 		break
 	default:
-		return errors.Errorf("invalid provider type. Allowed values are [%s, %s, %s, %s]",
+		return errors.Errorf("invalid provider type. Allowed values are [%s, %s, %s, %s, %s, %s]",
 			clusterctlv1.CoreProviderType,
 			clusterctlv1.BootstrapProviderType,
 			clusterctlv1.InfrastructureProviderType,
-			clusterctlv1.ControlPlaneProviderType)
+			clusterctlv1.ControlPlaneProviderType,
+			clusterctlv1.IPAMProviderType,
+			clusterctlv1.RuntimeExtensionProviderType)
 	}
 	return nil
 }

cmd/clusterctl/client/delete.go

@@ -30,21 +30,24 @@ type DeleteOptions struct {
 	// default rules for kubeconfig discovery will be used.
 	Kubeconfig Kubeconfig
 
-	// CoreProvider version (e.g. cluster-api:v1.1.5) to add to the management cluster. If unspecified, the
-	// cluster-api core provider's latest release is used.
+	// CoreProvider version (e.g. cluster-api:v1.1.5) to delete from the management cluster.
 	CoreProvider string
 
-	// BootstrapProviders and versions (e.g. kubeadm:v1.1.5) to add to the management cluster.
-	// If unspecified, the kubeadm bootstrap provider's latest release is used.
+	// BootstrapProviders and versions (e.g. kubeadm:v1.1.5) to delete from the management cluster.
 	BootstrapProviders []string
 
-	// InfrastructureProviders and versions (e.g. aws:v0.5.0) to add to the management cluster.
+	// InfrastructureProviders and versions (e.g. aws:v0.5.0) to delete from the management cluster.
 	InfrastructureProviders []string
 
-	// ControlPlaneProviders and versions (e.g. kubeadm:v1.1.5) to add to the management cluster.
-	// If unspecified, the kubeadm control plane provider latest release is used.
+	// ControlPlaneProviders and versions (e.g. kubeadm:v1.1.5) to delete from the management cluster.
 	ControlPlaneProviders []string
 
+	// IPAMProviders and versions (e.g. infoblox:v0.0.1) to delete from the management cluster.
+	IPAMProviders []string
+
+	// RuntimeExtensionProviders and versions (e.g. test:v0.0.1) to delete from the management cluster.
+	RuntimeExtensionProviders []string
+
 	// DeleteAll set for deletion of all the providers.
 	DeleteAll bool
 
@@ -109,6 +112,16 @@ func (c *clusterctlClient) Delete(options DeleteOptions) error {
 			return err
 		}
 
+		providers, err = appendProviders(providers, clusterctlv1.IPAMProviderType, options.IPAMProviders...)
+		if err != nil {
+			return err
+		}
+
+		providers, err = appendProviders(providers, clusterctlv1.RuntimeExtensionProviderType, options.RuntimeExtensionProviders...)
+		if err != nil {
+			return err
+		}
+
 		for _, provider := range providers {
 			// Try to detect the namespace where the provider lives
 			provider.Namespace, err = clusterClient.ProviderInventory().GetProviderNamespace(provider.ProviderName, provider.GetProviderType())

cmd/clusterctl/client/init.go

@@ -53,6 +53,12 @@ type InitOptions struct {
 	// If unspecified, the kubeadm control plane provider latest release is used.
 	ControlPlaneProviders []string
 
+	// IPAMProviders and versions (e.g. infoblox:v0.0.1) to add to the management cluster.
+	IPAMProviders []string
+
+	// RuntimeExtensionProviders and versions (e.g. test:v0.0.1) to add to the management cluster.
+	RuntimeExtensionProviders []string
+
 	// TargetNamespace defines the namespace where the providers should be deployed. If unspecified, each provider
 	// will be installed in a provider's default namespace.
 	TargetNamespace string
@@ -227,6 +233,14 @@ func (c *clusterctlClient) setupInstaller(cluster cluster.Client, options InitOp
 		return nil, err
 	}
 
+	if err := c.addToInstaller(addOptions, clusterctlv1.IPAMProviderType, options.IPAMProviders...); err != nil {
+		return nil, err
+	}
+
+	if err := c.addToInstaller(addOptions, clusterctlv1.RuntimeExtensionProviderType, options.RuntimeExtensionProviders...); err != nil {
+		return nil, err
+	}
+
 	return installer, nil
 }
 

cmd/clusterctl/client/upgrade.go

@@ -109,6 +109,12 @@ type ApplyUpgradeOptions struct {
 	// InfrastructureProviders instance and versions (e.g. capa-system/aws:v0.5.0) to upgrade to. This field can be used as alternative to Contract.
 	InfrastructureProviders []string
 
+	// IPAMProviders instance and versions (e.g. ipam-system/infoblox:v0.0.1) to upgrade to. This field can be used as alternative to Contract.
+	IPAMProviders []string
+
+	// RuntimeExtensionProviders instance and versions (e.g. runtime-extension-system/test:v0.0.1) to upgrade to. This field can be used as alternative to Contract.
+	RuntimeExtensionProviders []string
+
 	// WaitProviders instructs the upgrade apply command to wait till the providers are successfully upgraded.
 	WaitProviders bool
 
@@ -155,7 +161,9 @@ func (c *clusterctlClient) ApplyUpgrade(options ApplyUpgradeOptions) error {
 	isCustomUpgrade := options.CoreProvider != "" ||
 		len(options.BootstrapProviders) > 0 ||
 		len(options.ControlPlaneProviders) > 0 ||
-		len(options.InfrastructureProviders) > 0
+		len(options.InfrastructureProviders) > 0 ||
+		len(options.IPAMProviders) > 0 ||
+		len(options.RuntimeExtensionProviders) > 0
 
 	opts := cluster.UpgradeOptions{
 		WaitProviders:       options.WaitProviders,
@@ -185,6 +193,14 @@ func (c *clusterctlClient) ApplyUpgrade(options ApplyUpgradeOptions) error {
 		if err != nil {
 			return err
 		}
+		upgradeItems, err = addUpgradeItems(upgradeItems, clusterctlv1.IPAMProviderType, options.IPAMProviders...)
+		if err != nil {
+			return err
+		}
+		upgradeItems, err = addUpgradeItems(upgradeItems, clusterctlv1.RuntimeExtensionProviderType, options.RuntimeExtensionProviders...)
+		if err != nil {
+			return err
+		}
 
 		// Execute the upgrade using the custom upgrade items
 		return clusterClient.ProviderUpgrader().ApplyCustomPlan(opts, upgradeItems...)