feat: use registryMirror addon as Containerd mirror

Author: dkoshkin

pkg/handlers/generic/mutation/mirrors/inject.go

@@ -11,6 +11,7 @@ import (
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
+	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
 	controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1"
 	runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1"
@@ -22,6 +23,7 @@ import (
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/patches"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/patches/selectors"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/variables"
+	registrymirrorutils "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/registrymirror/utils"
 	handlersutils "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/utils"
 )
 
@@ -62,8 +64,8 @@ func (h *globalMirrorPatchHandler) Mutate(
 	obj *unstructured.Unstructured,
 	vars map[string]apiextensionsv1.JSON,
 	holderRef runtimehooksv1.HolderReference,
-	clusterKey ctrlclient.ObjectKey,
-	_ mutation.ClusterGetter,
+	_ ctrlclient.ObjectKey,
+	clusterGetter mutation.ClusterGetter,
 ) error {
 	log := ctrl.LoggerFrom(ctx).WithValues(
 		"holderRef", holderRef,
@@ -82,14 +84,24 @@ func (h *globalMirrorPatchHandler) Mutate(
 		v1alpha1.ImageRegistriesVariableName,
 	)
 
+	_, registryMirrorErr := variables.Get[v1alpha1.RegistryMirror](
+		vars,
+		v1alpha1.ClusterConfigVariableName,
+		[]string{"addons", v1alpha1.RegistryMirrorVariableName}...)
+
 	switch {
-	case variables.IsNotFoundError(imageRegistriesErr) && variables.IsNotFoundError(globalMirrorErr):
-		log.V(5).Info("Image Registry Credentials and Global Registry Mirror variable not defined")
+	case variables.IsNotFoundError(imageRegistriesErr) &&
+		variables.IsNotFoundError(globalMirrorErr) &&
+		variables.IsNotFoundError(registryMirrorErr):
+		log.V(5).
+			Info("Image Registry Credentials and Global Registry Mirror and Registry Mirror Addon variable not defined")
 		return nil
 	case imageRegistriesErr != nil && !variables.IsNotFoundError(imageRegistriesErr):
 		return imageRegistriesErr
 	case globalMirrorErr != nil && !variables.IsNotFoundError(globalMirrorErr):
 		return globalMirrorErr
+	case registryMirrorErr != nil && !variables.IsNotFoundError(registryMirrorErr):
+		return registryMirrorErr
 	}
 
 	var registriesWithOptionalCA []containerdConfig //nolint:prealloc // We don't know the size of the slice yet.
@@ -121,6 +133,26 @@ func (h *globalMirrorPatchHandler) Mutate(
 			registryWithOptionalCredentials,
 		)
 	}
+	if registryMirrorErr == nil {
+		cluster, err := clusterGetter(ctx)
+		if err != nil {
+			log.Error(
+				err,
+				"failed to get cluster from Global Mirror mutation handler",
+			)
+			return err
+		}
+
+		registryConfig, err := containerdConfigFromRegistryMirrorAddon(
+			ctx,
+			h.client,
+			cluster,
+		)
+		if err != nil {
+			return err
+		}
+		registriesWithOptionalCA = append(registriesWithOptionalCA, registryConfig)
+	}
 
 	needConfiguration := needContainerdConfiguration(
 		registriesWithOptionalCA,
@@ -234,6 +266,25 @@ func containerdConfigFromImageRegistry(
 	return configWithOptionalCACert, nil
 }
 
+func containerdConfigFromRegistryMirrorAddon(
+	_ context.Context,
+	_ ctrlclient.Client,
+	cluster *clusterv1.Cluster,
+) (containerdConfig, error) {
+	serviceIP, err := registrymirrorutils.ServiceIPForCluster(cluster)
+	if err != nil {
+		return containerdConfig{}, fmt.Errorf("error getting service IP for the registry mirror addon: %w", err)
+	}
+
+	config := containerdConfig{
+		// FIXME: Generate a self-signed CA.
+		URL:    fmt.Sprintf("http://%s", serviceIP),
+		Mirror: true,
+	}
+
+	return config, nil
+}
+
 func generateFiles(
 	registriesWithOptionalCA []containerdConfig,
 ) ([]bootstrapv1.File, error) {

pkg/handlers/generic/mutation/mirrors/inject_test.go

@@ -11,7 +11,11 @@ import (
 	"github.com/stretchr/testify/assert"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apiserver/pkg/storage/names"
+	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1"
 
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1"
@@ -32,6 +36,10 @@ func TestMirrorsPatch(t *testing.T) {
 }
 
 var _ = Describe("Generate Global mirror patches", func() {
+	clientScheme := runtime.NewScheme()
+	utilruntime.Must(clientgoscheme.AddToScheme(clientScheme))
+	utilruntime.Must(clusterv1.AddToScheme(clientScheme))
+
 	patchGenerator := func() mutation.GeneratePatches {
 		// Always initialize the testEnv variable in the closure.
 		// This will allow ginkgo to initialize testEnv variable during test execution time.
@@ -40,7 +48,7 @@ var _ = Describe("Generate Global mirror patches", func() {
 		// that are written by the tests.
 		// Test cases writes credentials secret that the mutator handler reads.
 		// Using direct client will enable reading it immediately.
-		client, err := testEnv.GetK8sClient()
+		client, err := testEnv.GetK8sClientWithScheme(clientScheme)
 		gomega.Expect(err).To(gomega.BeNil())
 		return mutation.NewMetaGeneratePatchesHandler("", client, NewPatch(client)).(mutation.GeneratePatches)
 	}
@@ -330,11 +338,69 @@ var _ = Describe("Generate Global mirror patches", func() {
 				},
 			},
 		},
+		{
+			Name: "files added in KubeadmControlPlaneTemplate for registry mirror addon",
+			Vars: []runtimehooksv1.Variable{
+				capitest.VariableWithValue(
+					v1alpha1.ClusterConfigVariableName,
+					v1alpha1.RegistryMirror{},
+					[]string{"addons", v1alpha1.RegistryMirrorVariableName}...,
+				),
+			},
+			RequestItem: request.NewKubeadmControlPlaneTemplateRequestItem(""),
+			ExpectedPatchMatchers: []capitest.JSONPatchMatcher{
+				{
+					Operation: "add",
+					Path:      "/spec/template/spec/kubeadmConfigSpec/files",
+					ValueMatcher: gomega.HaveExactElements(
+						gomega.HaveKeyWithValue(
+							"path", "/etc/containerd/certs.d/_default/hosts.toml",
+						),
+						gomega.HaveKeyWithValue(
+							"path", "/etc/caren/containerd/patches/registry-config.toml",
+						),
+					),
+				},
+			},
+		},
+		{
+			Name: "files added in KubeadmConfigTemplate for registry mirror addon",
+			Vars: []runtimehooksv1.Variable{
+				capitest.VariableWithValue(
+					v1alpha1.ClusterConfigVariableName,
+					v1alpha1.RegistryMirror{},
+					[]string{"addons", v1alpha1.RegistryMirrorVariableName}...,
+				),
+				capitest.VariableWithValue(
+					"builtin",
+					map[string]any{
+						"machineDeployment": map[string]any{
+							"class": names.SimpleNameGenerator.GenerateName("worker-"),
+						},
+					},
+				),
+			},
+			RequestItem: request.NewKubeadmConfigTemplateRequestItem(""),
+			ExpectedPatchMatchers: []capitest.JSONPatchMatcher{
+				{
+					Operation: "add",
+					Path:      "/spec/template/spec/files",
+					ValueMatcher: gomega.HaveExactElements(
+						gomega.HaveKeyWithValue(
+							"path", "/etc/containerd/certs.d/_default/hosts.toml",
+						),
+						gomega.HaveKeyWithValue(
+							"path", "/etc/caren/containerd/patches/registry-config.toml",
+						),
+					),
+				},
+			},
+		},
 	}
 
 	// Create credentials secret before each test
 	BeforeEach(func(ctx SpecContext) {
-		client, err := helpers.TestEnv.GetK8sClient()
+		client, err := helpers.TestEnv.GetK8sClientWithScheme(clientScheme)
 		gomega.Expect(err).To(gomega.BeNil())
 		gomega.Expect(client.Create(
 			ctx,
@@ -344,11 +410,21 @@ var _ = Describe("Generate Global mirror patches", func() {
 			ctx,
 			newMirrorSecretWithoutCA(validMirrorNoCASecretName, request.Namespace),
 		)).To(gomega.BeNil())
+
+		gomega.Expect(client.Create(
+			ctx,
+			&clusterv1.Cluster{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      request.ClusterName,
+					Namespace: request.Namespace,
+				},
+			},
+		)).To(gomega.BeNil())
 	})
 
 	// Delete credentials secret after each test
 	AfterEach(func(ctx SpecContext) {
-		client, err := helpers.TestEnv.GetK8sClient()
+		client, err := helpers.TestEnv.GetK8sClientWithScheme(clientScheme)
 		gomega.Expect(err).To(gomega.BeNil())
 		gomega.Expect(client.Delete(
 			ctx,
@@ -358,6 +434,16 @@ var _ = Describe("Generate Global mirror patches", func() {
 			ctx,
 			newMirrorSecretWithoutCA(validMirrorNoCASecretName, request.Namespace),
 		)).To(gomega.BeNil())
+
+		gomega.Expect(client.Delete(
+			ctx,
+			&clusterv1.Cluster{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      request.ClusterName,
+					Namespace: request.Namespace,
+				},
+			},
+		)).To(gomega.BeNil())
 	})
 
 	// create test node for each case