capiinstaller: Read manifests direct from payload image

Author: mdbooth

cmd/cluster-capi-operator/main.go

@@ -63,13 +63,20 @@ import (
 	"github.com/openshift/cluster-capi-operator/pkg/controllers/kubeconfig"
 	"github.com/openshift/cluster-capi-operator/pkg/controllers/secretsync"
 	"github.com/openshift/cluster-capi-operator/pkg/operatorstatus"
+	"github.com/openshift/cluster-capi-operator/pkg/providerimages"
 	"github.com/openshift/cluster-capi-operator/pkg/util"
 	"github.com/openshift/cluster-capi-operator/pkg/webhook"
 )
 
 const (
-	defaultImagesLocation      = "./dev-images.json"
+	defaultImagesLocation = "./dev-images.json"
+
 	defaultMachineAPINamespace = "openshift-machine-api"
+
+	pullSecretPathEnvVar        = "PULL_SECRET_PATH"
+	defaultPullSecretPath       = "/var/run/secrets/pull-secret/config.json"
+	providerImageDirEnvVar      = "PROVIDER_IMAGE_DIR"
+	defaultProviderImageDirPath = "/var/lib/provider-images"
 )
 
 func initScheme(scheme *runtime.Scheme) {
@@ -208,6 +215,28 @@ func main() {
 		os.Exit(1)
 	}
 
+	pullSecretPath := os.Getenv(pullSecretPathEnvVar)
+	if pullSecretPath == "" {
+		pullSecretPath = defaultPullSecretPath
+	}
+
+	providerImageDir := os.Getenv(providerImageDirEnvVar)
+	if providerImageDir == "" {
+		providerImageDir = defaultProviderImageDirPath
+	}
+
+	pullSecret, err := os.ReadFile(pullSecretPath)
+	if err != nil {
+		klog.Error(err, "unable to read pull secret", "path", pullSecretPath)
+		os.Exit(1)
+	}
+
+	providerImages, err := providerimages.ReadProviderImages(context.Background(), containerImages, providerImageDir, pullSecret)
+	if err != nil {
+		klog.Error(err, "unable to get provider image metadata")
+		os.Exit(1)
+	}
+
 	infra, err := util.GetInfra(context.Background(), mgr.GetAPIReader())
 	if err != nil {
 		klog.Error(err, "unable to get infrastructure object")
@@ -220,7 +249,7 @@ func main() {
 		os.Exit(1)
 	}
 
-	setupPlatformReconcilers(mgr, infra, platform, containerImages, applyClient, apiextensionsClient, *managedNamespace)
+	setupPlatformReconcilers(mgr, infra, platform, containerImages, providerImages, applyClient, apiextensionsClient, *managedNamespace)
 
 	// +kubebuilder:scaffold:builder
 
@@ -252,17 +281,17 @@ func getClusterOperatorStatusClient(mgr manager.Manager, controller string, plat
 	}
 }
 
-func setupPlatformReconcilers(mgr manager.Manager, infra *configv1.Infrastructure, platform configv1.PlatformType, containerImages map[string]string, applyClient *kubernetes.Clientset, apiextensionsClient *apiextensionsclient.Clientset, managedNamespace string) {
+func setupPlatformReconcilers(mgr manager.Manager, infra *configv1.Infrastructure, platform configv1.PlatformType, containerImages map[string]string, providerImages map[string]providerimages.ProviderImageManifests, applyClient *kubernetes.Clientset, apiextensionsClient *apiextensionsclient.Clientset, managedNamespace string) {
 	// Only setup reconcile controllers and webhooks when the platform is supported.
 	// This avoids unnecessary CAPI providers discovery, installs and reconciles when the platform is not supported.
 	isUnsupportedPlatform := false
 
 	switch platform {
 	case configv1.AWSPlatformType:
-		setupReconcilers(mgr, infra, platform, &awsv1.AWSCluster{}, containerImages, applyClient, apiextensionsClient, managedNamespace)
+		setupReconcilers(mgr, infra, platform, &awsv1.AWSCluster{}, containerImages, providerImages, applyClient, apiextensionsClient, managedNamespace)
 		setupWebhooks(mgr)
 	case configv1.GCPPlatformType:
-		setupReconcilers(mgr, infra, platform, &gcpv1.GCPCluster{}, containerImages, applyClient, apiextensionsClient, managedNamespace)
+		setupReconcilers(mgr, infra, platform, &gcpv1.GCPCluster{}, containerImages, providerImages, applyClient, apiextensionsClient, managedNamespace)
 		setupWebhooks(mgr)
 	case configv1.AzurePlatformType:
 		azureCloudEnvironment := getAzureCloudEnvironment(infra.Status.PlatformStatus)
@@ -272,20 +301,20 @@ func setupPlatformReconcilers(mgr manager.Manager, infra *configv1.Infrastructur
 			isUnsupportedPlatform = true
 		} else {
 			// The ClusterOperator Controller must run in all cases.
-			setupReconcilers(mgr, infra, platform, &azurev1.AzureCluster{}, containerImages, applyClient, apiextensionsClient, managedNamespace)
+			setupReconcilers(mgr, infra, platform, &azurev1.AzureCluster{}, containerImages, providerImages, applyClient, apiextensionsClient, managedNamespace)
 			setupWebhooks(mgr)
 		}
 	case configv1.PowerVSPlatformType:
-		setupReconcilers(mgr, infra, platform, &ibmpowervsv1.IBMPowerVSCluster{}, containerImages, applyClient, apiextensionsClient, managedNamespace)
+		setupReconcilers(mgr, infra, platform, &ibmpowervsv1.IBMPowerVSCluster{}, containerImages, providerImages, applyClient, apiextensionsClient, managedNamespace)
 		setupWebhooks(mgr)
 	case configv1.VSpherePlatformType:
-		setupReconcilers(mgr, infra, platform, &vspherev1.VSphereCluster{}, containerImages, applyClient, apiextensionsClient, managedNamespace)
+		setupReconcilers(mgr, infra, platform, &vspherev1.VSphereCluster{}, containerImages, providerImages, applyClient, apiextensionsClient, managedNamespace)
 		setupWebhooks(mgr)
 	case configv1.OpenStackPlatformType:
-		setupReconcilers(mgr, infra, platform, &openstackv1.OpenStackCluster{}, containerImages, applyClient, apiextensionsClient, managedNamespace)
+		setupReconcilers(mgr, infra, platform, &openstackv1.OpenStackCluster{}, containerImages, providerImages, applyClient, apiextensionsClient, managedNamespace)
 		setupWebhooks(mgr)
 	case configv1.BareMetalPlatformType:
-		setupReconcilers(mgr, infra, platform, &metal3v1.Metal3Cluster{}, containerImages, applyClient, apiextensionsClient, managedNamespace)
+		setupReconcilers(mgr, infra, platform, &metal3v1.Metal3Cluster{}, containerImages, providerImages, applyClient, apiextensionsClient, managedNamespace)
 		setupWebhooks(mgr)
 	default:
 		klog.Infof("Detected platform %q is not supported, skipping capi controllers setup", platform)
@@ -297,7 +326,7 @@ func setupPlatformReconcilers(mgr manager.Manager, infra *configv1.Infrastructur
 	setupClusterOperatorController(mgr, platform, managedNamespace, isUnsupportedPlatform)
 }
 
-func setupReconcilers(mgr manager.Manager, infra *configv1.Infrastructure, platform configv1.PlatformType, infraClusterObject client.Object, containerImages map[string]string, applyClient *kubernetes.Clientset, apiextensionsClient *apiextensionsclient.Clientset, managedNamespace string) {
+func setupReconcilers(mgr manager.Manager, infra *configv1.Infrastructure, platform configv1.PlatformType, infraClusterObject client.Object, containerImages map[string]string, providerImages map[string]providerimages.ProviderImageManifests, applyClient *kubernetes.Clientset, apiextensionsClient *apiextensionsclient.Clientset, managedNamespace string) {
 	if err := (&corecluster.CoreClusterController{
 		ClusterOperatorStatusClient: getClusterOperatorStatusClient(mgr, "cluster-capi-operator-cluster-resource-controller", platform, managedNamespace),
 		Cluster:                     &clusterv1.Cluster{},
@@ -333,6 +362,7 @@ func setupReconcilers(mgr manager.Manager, infra *configv1.Infrastructure, platf
 		Platform:                    platform,
 		ApplyClient:                 applyClient,
 		APIExtensionsClient:         apiextensionsClient,
+		ProviderImages:              providerImages,
 	}).SetupWithManager(mgr); err != nil {
 		klog.Error(err, "unable to create capi installer controller", "controller", "CAPIInstaller")
 		os.Exit(1)

go.mod

@@ -17,7 +17,7 @@ require (
 	github.com/google/go-cmp v0.7.0
 	github.com/google/uuid v1.6.0
 	github.com/gophercloud/gophercloud/v2 v2.9.0
-	github.com/klauspost/compress v1.18.0
+	github.com/klauspost/compress v1.18.1
 	github.com/metal3-io/cluster-api-provider-metal3/api v1.11.2
 	github.com/onsi/ginkgo/v2 v2.27.2
 	github.com/onsi/gomega v1.38.2
@@ -27,7 +27,7 @@ require (
 	github.com/openshift/library-go v0.0.0-20251112091634-ab97ebb73f0f
 	github.com/pkg/errors v0.9.1
 	github.com/spf13/pflag v1.0.10
-	golang.org/x/tools v0.38.0
+	golang.org/x/tools v0.39.0
 	gopkg.in/yaml.v2 v2.4.0
 	k8s.io/api v0.34.1
 	k8s.io/apiextensions-apiserver v0.34.1
@@ -154,6 +154,7 @@ require (
 	github.com/google/btree v1.1.3 // indirect
 	github.com/google/cel-go v0.26.0 // indirect
 	github.com/google/gnostic-models v0.7.0 // indirect
+	github.com/google/go-containerregistry v0.20.7 // indirect
 	github.com/google/pprof v0.0.0-20250820193118-f64d9cf942d6 // indirect
 	github.com/gophercloud/utils/v2 v2.0.0-20241220104409-2e0af06694a1 // indirect
 	github.com/gordonklaus/ineffassign v0.1.0 // indirect
@@ -291,7 +292,7 @@ require (
 	golang.org/x/exp/typeparams v0.0.0-20250210185358-939b2ce775ac // indirect
 	golang.org/x/mod v0.30.0 // indirect
 	golang.org/x/net v0.47.0 // indirect
-	golang.org/x/oauth2 v0.32.0 // indirect
+	golang.org/x/oauth2 v0.33.0 // indirect
 	golang.org/x/sync v0.18.0 // indirect
 	golang.org/x/sys v0.38.0 // indirect
 	golang.org/x/term v0.37.0 // indirect

manifests/0000_30_cluster-api_11_deployment.yaml

@@ -35,6 +35,10 @@ spec:
         env:
         - name: RELEASE_VERSION
           value: "0.0.1-snapshot"
+        - name: PULL_SECRET_PATH
+          value: "/var/run/secrets/pull-secret/config.json"
+        - name: PROVIDER_IMAGE_DIR
+          value: "/var/lib/provider-images"
         ports:
         - containerPort: 9443
           name: webhook-server
@@ -53,6 +57,11 @@ spec:
         - name: cert
           mountPath: /tmp/k8s-webhook-server/serving-certs
           readOnly: true
+        - name: pull-secret
+          mountPath: /var/run/secrets/pull-secret
+          readOnly: true
+        - name: provider-images
+          mountPath: /var/lib/provider-images
       - name: machine-api-migration
         image: registry.ci.openshift.org/openshift:cluster-capi-operator
         command:
@@ -88,3 +97,9 @@ spec:
         secret:
           defaultMode: 420
           secretName: cluster-capi-operator-webhook-service-cert
+      - name: pull-secret
+        hostPath:
+          path: /var/lib/kubelet/config.json
+          type: File
+      - name: provider-images
+        emptyDir: {}

pkg/controllers/capiinstaller/capi_installer_controller.go

@@ -16,9 +16,13 @@ limitations under the License.
 package capiinstaller
 
 import (
+	"bytes"
 	"context"
 	"errors"
 	"fmt"
+	"io"
+	"maps"
+	"os"
 	"regexp"
 	"strings"
 
@@ -35,7 +39,6 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/serializer"
 	"k8s.io/utils/clock"
-	"k8s.io/utils/strings/slices"
 
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
@@ -47,6 +50,7 @@ import (
 	configv1 "github.com/openshift/api/config/v1"
 	"github.com/openshift/cluster-capi-operator/pkg/metadata"
 	"github.com/openshift/cluster-capi-operator/pkg/operatorstatus"
+	"github.com/openshift/cluster-capi-operator/pkg/providerimages"
 	"github.com/openshift/library-go/pkg/operator/events"
 	"github.com/openshift/library-go/pkg/operator/resource/resourceapply"
 	"github.com/openshift/library-go/pkg/operator/resource/resourcemerge"
@@ -89,6 +93,7 @@ type CapiInstallerController struct {
 	Platform            configv1.PlatformType
 	ApplyClient         *kubernetes.Clientset
 	APIExtensionsClient *apiextensionsclient.Clientset
+	ProviderImages      map[string]providerimages.ProviderImageManifests
 }
 
 // Reconcile reconciles the cluster-api ClusterOperator object.
@@ -185,6 +190,38 @@ func (r *CapiInstallerController) reconcile(ctx context.Context, log logr.Logger
 		log.Info("finished reconciling CAPI provider", "name", name)
 	}
 
+	providerImages := func(yield func(providerImage providerimages.ProviderImageManifests) bool) {
+		for providerImage := range maps.Values(r.ProviderImages) {
+			if providerImage.Type == "core" {
+				if !yield(providerImage) {
+					return
+				}
+			}
+
+			if providerImage.Type == "infrastructure" && providerImage.OCPPlatform == string(r.Platform) {
+				if !yield(providerImage) {
+					return
+				}
+			}
+		}
+	}
+
+	for providerImage := range providerImages {
+		reader, err := providerManifestReader(providerImage)
+		if err != nil {
+			return fmt.Errorf("failed to create provider manifest reader: %w", err)
+		}
+
+		yamlManifests, err := extractManifests(reader)
+		if err != nil {
+			return fmt.Errorf("failed to extract manifests from provider manifest: %w", err)
+		}
+
+		if err := r.applyProviderComponents(ctx, yamlManifests); err != nil {
+			return fmt.Errorf("failed to apply provider components: %w", err)
+		}
+	}
+
 	return nil
 }
 
@@ -283,6 +320,11 @@ func getProviderComponents(scheme *runtime.Scheme, components []string) ([]strin
 	customResourceDefinitionAssets := make(map[string]string)
 
 	for i, m := range components {
+		// Skip empty manifests.
+		if strings.TrimSpace(m) == "" {
+			continue
+		}
+
 		// Parse the YAML manifests into unstructure objects.
 		u, err := yamlToUnstructured(scheme, m)
 		if err != nil {
@@ -409,7 +451,12 @@ func (r *CapiInstallerController) SetupWithManager(mgr ctrl.Manager) error {
 // clusterctl Provider Contract - Components YAML file contract defined at:
 // https://github.com/kubernetes-sigs/cluster-api/blob/a36712e28bf5d54e398ea84cb3e20102c0499426/docs/book/src/clusterctl/provider-contract.md?plain=1#L157-L162
 func (r *CapiInstallerController) extractProviderComponents(cm corev1.ConfigMap) ([]string, error) {
-	yamlManifests, err := extractManifests(cm)
+	reader, err := configMapReader(cm)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create config map reader: %w", err)
+	}
+
+	yamlManifests, err := extractManifests(reader)
 	if err != nil {
 		return nil, fmt.Errorf("failed to extract manifests from configMap: %w", err)
 	}
@@ -427,46 +474,41 @@ func (r *CapiInstallerController) extractProviderComponents(cm corev1.ConfigMap)
 	return replacedYamlManifests, nil
 }
 
-// extractManifests extracts and processes component manifests from given ConfigMap.
-// If the data is in compressed binary form, it decompresses them.
-func extractManifests(cm corev1.ConfigMap) ([]string, error) {
-	data, hasData := cm.Data["components"]
-	binaryData, hasBinary := cm.BinaryData["components-zstd"]
+func configMapReader(cm corev1.ConfigMap) (io.Reader, error) {
+	if data, ok := cm.Data["components"]; ok {
+		return strings.NewReader(data), nil
+	}
 
-	if !(hasBinary || hasData) {
-		return nil, errEmptyProviderConfigMap
+	if binaryData, ok := cm.BinaryData["components-zstd"]; ok {
+		return zstd.NewReader(bytes.NewReader(binaryData))
 	}
 
-	if hasBinary {
-		decoder, err := zstd.NewReader(nil)
-		if err != nil {
-			return nil, fmt.Errorf("failed to create zstd reader: %w", err)
-		}
+	return nil, errEmptyProviderConfigMap
+}
 
-		decoded, err := decoder.DecodeAll(binaryData, []byte{})
-		if err != nil {
-			return nil, fmt.Errorf("failed to decompress components: %w", err)
-		}
+func providerManifestReader(providerImage providerimages.ProviderImageManifests) (io.Reader, error) {
+	return os.Open(providerImage.ManifestsPath)
+}
 
-		data = string(decoded)
+// extractManifests extracts and processes component manifests from given ConfigMap.
+// If the data is in compressed binary form, it decompresses them.
+func extractManifests(reader io.Reader) ([]string, error) {
+	data, err := io.ReadAll(reader)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read components: %w", err)
 	}
 
 	// Certain provider components have drone/envsubst environment variables interpolated within the manifest.
 	// Substitute them with the value defined in the environment variable (see setFeatureGatesEnvVars()).
 	// If that's not set, fallback to the default value defined in the template.
-	components, err := envsubst.EvalEnv(data)
+	components, err := envsubst.EvalEnv(string(data))
 	if err != nil {
 		return nil, fmt.Errorf("failed to substitute environment variables in component manifests: %w", err)
 	}
 
 	// Split multi-document YAML into single manifests.
 	yamlManifests := regexp.MustCompile("(?m)^---$").Split(components, -1)
 
-	// Filter out empty manifests, e.g. when a bundle starts with '---'
-	yamlManifests = slices.Filter(nil, yamlManifests, func(m string) bool {
-		return strings.TrimSpace(m) != ""
-	})
-
 	return yamlManifests, nil
 }
 

pkg/controllers/capiinstaller/capi_installer_controller_test.go

@@ -93,7 +93,12 @@ var _ = Describe("extractManifests", func() {
 
 	for _, tc := range testCases {
 		It(tc.name, func() {
-			manifests, err := extractManifests(tc.configMap)
+			reader, err := configMapReader(tc.configMap)
+			if err != nil {
+				Expect(err).To(MatchError(errEmptyProviderConfigMap))
+			}
+
+			manifests, err := extractManifests(reader)
 
 			if tc.expectedError != nil {
 				Expect(err).To(MatchError(tc.expectedError))