diff --git a/make/dev.mk b/make/dev.mk index 5d19fd458..46b2d97d8 100644 --- a/make/dev.mk +++ b/make/dev.mk @@ -32,7 +32,7 @@ dev.update-webhook-image-on-kind: kind load docker-image --name $(KIND_CLUSTER_NAME) \ ko.local/cluster-api-runtime-extensions-nutanix:$(SNAPSHOT_VERSION) kubectl set image deployment \ - cluster-api-runtime-extensions-nutanix webhook=ko.local/cluster-api-runtime-extensions-nutanix:$(SNAPSHOT_VERSION) + cluster-api-runtime-extensions-nutanix manager=ko.local/cluster-api-runtime-extensions-nutanix:$(SNAPSHOT_VERSION) kubectl rollout restart deployment cluster-api-runtime-extensions-nutanix kubectl rollout status deployment cluster-api-runtime-extensions-nutanix diff --git a/pkg/handlers/deleteinv0280/generic/mutation/handlers.go b/pkg/handlers/deleteinv0280/generic/mutation/handlers.go index 9ccafedfe..324611ccb 100644 --- a/pkg/handlers/deleteinv0280/generic/mutation/handlers.go +++ b/pkg/handlers/deleteinv0280/generic/mutation/handlers.go @@ -8,6 +8,7 @@ import ( "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers/mutation" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/aws/mutation/cni/calico" + deleteinv0280credentials "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/deleteinv0280/generic/mutation/imageregistries/credentials" deleteinv0280mirrors "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/deleteinv0280/generic/mutation/mirrors" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/auditpolicy" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/autorenewcerts" @@ -19,7 +20,6 @@ import ( "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/etcd" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/extraapiservercertsans" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/httpproxy" - "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/imageregistries/credentials" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/kubernetesimagerepository" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/users" ) @@ -33,7 +33,7 @@ func MetaMutators(mgr manager.Manager) []mutation.MetaMutator { extraapiservercertsans.NewPatch(), httpproxy.NewPatch(mgr.GetClient()), kubernetesimagerepository.NewPatch(), - credentials.NewPatch(mgr.GetClient()), + deleteinv0280credentials.NewPatch(mgr.GetClient()), deleteinv0280mirrors.NewPatch(mgr.GetClient()), calico.NewPatch(), users.NewPatch(), diff --git a/pkg/handlers/deleteinv0280/generic/mutation/imageregistries/credentials/credential_provider_config_files.go b/pkg/handlers/deleteinv0280/generic/mutation/imageregistries/credentials/credential_provider_config_files.go new file mode 100644 index 000000000..ee15ca6b7 --- /dev/null +++ b/pkg/handlers/deleteinv0280/generic/mutation/imageregistries/credentials/credential_provider_config_files.go @@ -0,0 +1,264 @@ +// Copyright 2023 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package credentials + +import ( + "bytes" + _ "embed" + "fmt" + "net/url" + "path" + "text/template" + + corev1 "k8s.io/api/core/v1" + credentialproviderv1 "k8s.io/kubelet/pkg/apis/credentialprovider/v1" + cabpkv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" + + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/imageregistries/credentials/credentialprovider" +) + +const ( + //nolint:gosec // Does not contain hard coded credentials. + kubeletStaticCredentialProviderCredentialsOnRemote = "/etc/kubernetes/static-image-credentials.json" + + //nolint:gosec // Does not contain hard coded credentials. + kubeletImageCredentialProviderConfigOnRemote = "/etc/kubernetes/image-credential-provider-config.yaml" + + //nolint:gosec // Does not contain hard coded credentials. + kubeletDynamicCredentialProviderConfigOnRemote = "/etc/kubernetes/dynamic-credential-provider-config.yaml" + + azureCloudConfigFilePath = "/etc/kubernetes/azure.json" + + secretKeyForCACert = "ca.crt" +) + +var ( + //go:embed templates/dynamic-credential-provider-config.yaml.gotmpl + dynamicCredentialProviderConfigPatch []byte + + dynamicCredentialProviderConfigPatchTemplate = template.Must( + template.New("").Parse(string(dynamicCredentialProviderConfigPatch)), + ) + + //go:embed templates/kubelet-image-credential-provider-config.yaml.gotmpl + kubeletImageCredentialProviderConfigPatch []byte + + kubeletImageCredentialProviderConfigPatchTemplate = template.Must( + template.New("").Parse(string(kubeletImageCredentialProviderConfigPatch)), + ) +) + +type providerConfig struct { + URL string + Username string + Password string + HasCACert bool + Mirror bool +} + +func (c providerConfig) isCredentialsEmpty() bool { + return c.Username == "" && + c.Password == "" +} + +func (c providerConfig) requiresStaticCredentials() (bool, error) { + registryHostWithPath, err := c.registryHostWithPath() + if err != nil { + return false, fmt.Errorf( + "failed to get registry host with path: %w", + err, + ) + } + + knownRegistryProvider, err := credentialprovider.URLMatchesKnownRegistryProvider( + registryHostWithPath, + ) + if err != nil { + return false, fmt.Errorf( + "failed to check if registry matches a known registry provider: %w", + err, + ) + } + + // require static credentials if the registry provider is not known + return !knownRegistryProvider, nil +} + +func (c providerConfig) registryHostWithPath() (string, error) { + registryURL, err := url.ParseRequestURI(c.URL) + if err != nil { + return "", fmt.Errorf("failed parsing registry URL: %w", err) + } + + registryHostWithPath := registryURL.Host + if registryURL.Path != "" { + registryHostWithPath = path.Join(registryURL.Host, registryURL.Path) + } + + return registryHostWithPath, nil +} + +func templateFilesForImageCredentialProviderConfigs( + configs []providerConfig, +) ([]cabpkv1.File, error) { + var files []cabpkv1.File + + kubeletCredentialProviderConfigFile, err := templateKubeletCredentialProviderConfig(configs) + if err != nil { + return nil, err + } + if kubeletCredentialProviderConfigFile != nil { + files = append(files, *kubeletCredentialProviderConfigFile) + } + + kubeletDynamicCredentialProviderConfigFile, err := templateDynamicCredentialProviderConfig( + configs, + ) + if err != nil { + return nil, err + } + if kubeletDynamicCredentialProviderConfigFile != nil { + files = append(files, *kubeletDynamicCredentialProviderConfigFile) + } + + return files, nil +} + +func templateKubeletCredentialProviderConfig( + configs []providerConfig, +) (*cabpkv1.File, error) { + providerBinary, providerArgs, providerAPIVersion := kubeletCredentialProvider() + + // In addition to the globs already defined in the template, also include the user provided registries. + // + // This is needed to match registries with a port and/or a URL path. + // From https://kubernetes.io/docs/tasks/administer-cluster/kubelet-credential-provider/#configure-image-matching + registryHosts := make([]string, 0, len(configs)) + for _, config := range configs { + registryHostWithPath, err := config.registryHostWithPath() + if err != nil { + return nil, err + } + registryHosts = append(registryHosts, registryHostWithPath) + } + + templateInput := struct { + RegistryHosts []string + ProviderBinary string + ProviderArgs []string + ProviderAPIVersion string + }{ + RegistryHosts: registryHosts, + ProviderBinary: providerBinary, + ProviderArgs: providerArgs, + ProviderAPIVersion: providerAPIVersion, + } + + return fileFromTemplate( + kubeletImageCredentialProviderConfigPatchTemplate, + templateInput, + kubeletImageCredentialProviderConfigOnRemote, + ) +} + +func templateDynamicCredentialProviderConfig( + configs []providerConfig, +) (*cabpkv1.File, error) { + type templateInput struct { + RegistryHost string + ProviderBinary string + ProviderArgs []string + ProviderAPIVersion string + Mirror bool + } + + inputs := make([]templateInput, 0, len(configs)) + + for _, config := range configs { + registryHostWithPath, err := config.registryHostWithPath() + if err != nil { + return nil, err + } + + providerBinary, providerArgs, providerAPIVersion, err := dynamicCredentialProvider( + registryHostWithPath, + ) + if err != nil { + return nil, err + } + + inputs = append(inputs, templateInput{ + RegistryHost: registryHostWithPath, + ProviderBinary: providerBinary, + ProviderArgs: providerArgs, + ProviderAPIVersion: providerAPIVersion, + Mirror: config.Mirror, + }) + } + + return fileFromTemplate( + dynamicCredentialProviderConfigPatchTemplate, + inputs, + kubeletDynamicCredentialProviderConfigOnRemote, + ) +} + +func kubeletCredentialProvider() (providerBinary string, providerArgs []string, providerAPIVersion string) { + return "dynamic-credential-provider", + []string{"get-credentials", "-c", kubeletDynamicCredentialProviderConfigOnRemote}, + credentialproviderv1.SchemeGroupVersion.String() +} + +func dynamicCredentialProvider(host string) ( + providerBinary string, providerArgs []string, providerAPIVersion string, err error, +) { + if matches, err := credentialprovider.URLMatchesECR(host); matches || err != nil { + return "ecr-credential-provider", []string{"get-credentials"}, + credentialproviderv1.SchemeGroupVersion.String(), err + } + + if matches, err := credentialprovider.URLMatchesGCR(host); matches || err != nil { + return "gcr-credential-provider", []string{"get-credentials"}, + credentialproviderv1.SchemeGroupVersion.String(), err + } + + if matches, err := credentialprovider.URLMatchesACR(host); matches || err != nil { + return "acr-credential-provider", []string{ + azureCloudConfigFilePath, + }, credentialproviderv1.SchemeGroupVersion.String(), err + } + + // if no supported provider was found, assume we are using the static credential provider + return "static-credential-provider", + []string{kubeletStaticCredentialProviderCredentialsOnRemote}, + credentialproviderv1.SchemeGroupVersion.String(), + nil +} + +func fileFromTemplate( + t *template.Template, + templateInput any, + fPath string, +) (*cabpkv1.File, error) { + var b bytes.Buffer + err := t.Execute(&b, templateInput) + if err != nil { + return nil, fmt.Errorf("failed executing template: %w", err) + } + + return &cabpkv1.File{ + Path: fPath, + Content: b.String(), + Permissions: "0600", + }, nil +} + +func secretHasCACert(secret *corev1.Secret) bool { + if secret == nil { + return false + } + + _, ok := secret.Data[secretKeyForCACert] + return ok +} diff --git a/pkg/handlers/deleteinv0280/generic/mutation/imageregistries/credentials/credential_provider_config_files_test.go b/pkg/handlers/deleteinv0280/generic/mutation/imageregistries/credentials/credential_provider_config_files_test.go new file mode 100644 index 000000000..75647e5a0 --- /dev/null +++ b/pkg/handlers/deleteinv0280/generic/mutation/imageregistries/credentials/credential_provider_config_files_test.go @@ -0,0 +1,398 @@ +// Copyright 2023 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package credentials + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + cabpkv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" +) + +func Test_templateKubeletCredentialProviderConfig(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + credentials []providerConfig + want *cabpkv1.File + wantErr error + }{ + { + name: "ECR image registry", + credentials: []providerConfig{ + {URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com"}, + }, + want: &cabpkv1.File{ + Path: "/etc/kubernetes/image-credential-provider-config.yaml", + Owner: "", + Permissions: "0600", + Encoding: "", + Append: false, + Content: `apiVersion: kubelet.config.k8s.io/v1 +kind: CredentialProviderConfig +providers: +- name: dynamic-credential-provider + args: + - get-credentials + - -c + - /etc/kubernetes/dynamic-credential-provider-config.yaml + matchImages: + - "123456789.dkr.ecr.us-east-1.amazonaws.com" + - "*" + - "*.*" + - "*.*.*" + - "*.*.*.*" + - "*.*.*.*.*" + - "*.*.*.*.*.*" + defaultCacheDuration: "0s" + apiVersion: credentialprovider.kubelet.k8s.io/v1 +`, + }, + }, + { + name: "image registry with static config", + credentials: []providerConfig{{ + URL: "https://myregistry.com:5000/myproject", + Username: "myuser", + Password: "mypassword", + }}, + want: &cabpkv1.File{ + Path: "/etc/kubernetes/image-credential-provider-config.yaml", + Owner: "", + Permissions: "0600", + Encoding: "", + Append: false, + Content: `apiVersion: kubelet.config.k8s.io/v1 +kind: CredentialProviderConfig +providers: +- name: dynamic-credential-provider + args: + - get-credentials + - -c + - /etc/kubernetes/dynamic-credential-provider-config.yaml + matchImages: + - "myregistry.com:5000/myproject" + - "*" + - "*.*" + - "*.*.*" + - "*.*.*.*" + - "*.*.*.*.*" + - "*.*.*.*.*.*" + defaultCacheDuration: "0s" + apiVersion: credentialprovider.kubelet.k8s.io/v1 +`, + }, + }, + { + name: "docker.io registry with static credentials", + credentials: []providerConfig{{ + URL: "https://registry-1.docker.io", + Username: "myuser", + Password: "mypassword", + }}, + want: &cabpkv1.File{ + Path: "/etc/kubernetes/image-credential-provider-config.yaml", + Owner: "", + Permissions: "0600", + Encoding: "", + Append: false, + Content: `apiVersion: kubelet.config.k8s.io/v1 +kind: CredentialProviderConfig +providers: +- name: dynamic-credential-provider + args: + - get-credentials + - -c + - /etc/kubernetes/dynamic-credential-provider-config.yaml + matchImages: + - "registry-1.docker.io" + - "docker.io" + - "*" + - "*.*" + - "*.*.*" + - "*.*.*.*" + - "*.*.*.*.*" + - "*.*.*.*.*.*" + defaultCacheDuration: "0s" + apiVersion: credentialprovider.kubelet.k8s.io/v1 +`, + }, + }, + } + for idx := range tests { + tt := tests[idx] + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + file, err := templateKubeletCredentialProviderConfig(tt.credentials) + require.ErrorIs(t, err, tt.wantErr) + assert.Equal(t, tt.want, file) + }) + } +} + +func Test_templateDynamicCredentialProviderConfig(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + credentials []providerConfig + want *cabpkv1.File + wantErr error + }{ + { + name: "ECR image registry", + credentials: []providerConfig{ + {URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com"}, + }, + want: &cabpkv1.File{ + Path: "/etc/kubernetes/dynamic-credential-provider-config.yaml", + Owner: "", + Permissions: "0600", + Encoding: "", + Append: false, + Content: `apiVersion: credentialprovider.d2iq.com/v1alpha1 +kind: DynamicCredentialProviderConfig +credentialProviderPluginBinDir: /etc/kubernetes/image-credential-provider/ +credentialProviders: + apiVersion: kubelet.config.k8s.io/v1 + kind: CredentialProviderConfig + providers: + - name: ecr-credential-provider + args: + - get-credentials + matchImages: + - "123456789.dkr.ecr.us-east-1.amazonaws.com" + defaultCacheDuration: "0s" + apiVersion: credentialprovider.kubelet.k8s.io/v1 +`, + }, + }, + { + name: "image registry with static credentials", + credentials: []providerConfig{{ + URL: "https://myregistry.com:5000/myproject", + Username: "myuser", + Password: "mypassword", + }}, + want: &cabpkv1.File{ + Path: "/etc/kubernetes/dynamic-credential-provider-config.yaml", + Owner: "", + Permissions: "0600", + Encoding: "", + Append: false, + Content: `apiVersion: credentialprovider.d2iq.com/v1alpha1 +kind: DynamicCredentialProviderConfig +credentialProviderPluginBinDir: /etc/kubernetes/image-credential-provider/ +credentialProviders: + apiVersion: kubelet.config.k8s.io/v1 + kind: CredentialProviderConfig + providers: + - name: static-credential-provider + args: + - /etc/kubernetes/static-image-credentials.json + matchImages: + - "myregistry.com:5000/myproject" + defaultCacheDuration: "0s" + apiVersion: credentialprovider.kubelet.k8s.io/v1 +`, + }, + }, + { + name: "docker.io registry with static credentials", + credentials: []providerConfig{{ + URL: "https://registry-1.docker.io", + Username: "myuser", + Password: "mypassword", + }}, + want: &cabpkv1.File{ + Path: "/etc/kubernetes/dynamic-credential-provider-config.yaml", + Owner: "", + Permissions: "0600", + Encoding: "", + Append: false, + Content: `apiVersion: credentialprovider.d2iq.com/v1alpha1 +kind: DynamicCredentialProviderConfig +credentialProviderPluginBinDir: /etc/kubernetes/image-credential-provider/ +credentialProviders: + apiVersion: kubelet.config.k8s.io/v1 + kind: CredentialProviderConfig + providers: + - name: static-credential-provider + args: + - /etc/kubernetes/static-image-credentials.json + matchImages: + - "registry-1.docker.io" + - "docker.io" + defaultCacheDuration: "0s" + apiVersion: credentialprovider.kubelet.k8s.io/v1 +`, + }, + }, + { + name: "multiple registries", + credentials: []providerConfig{{ + URL: "https://registry-1.docker.io", + Username: "myuser", + Password: "mypassword", + }, { + URL: "https://myregistry.com", + Username: "myuser", + Password: "mypassword", + }, { + URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com", + }, { + URL: "https://anotherregistry.com", + Username: "anotheruser", + Password: "anotherpassword", + }}, + want: &cabpkv1.File{ + Path: "/etc/kubernetes/dynamic-credential-provider-config.yaml", + Owner: "", + Permissions: "0600", + Encoding: "", + Append: false, + Content: `apiVersion: credentialprovider.d2iq.com/v1alpha1 +kind: DynamicCredentialProviderConfig +credentialProviderPluginBinDir: /etc/kubernetes/image-credential-provider/ +credentialProviders: + apiVersion: kubelet.config.k8s.io/v1 + kind: CredentialProviderConfig + providers: + - name: static-credential-provider + args: + - /etc/kubernetes/static-image-credentials.json + matchImages: + - "registry-1.docker.io" + - "docker.io" + defaultCacheDuration: "0s" + apiVersion: credentialprovider.kubelet.k8s.io/v1 + - name: static-credential-provider + args: + - /etc/kubernetes/static-image-credentials.json + matchImages: + - "myregistry.com" + defaultCacheDuration: "0s" + apiVersion: credentialprovider.kubelet.k8s.io/v1 + - name: ecr-credential-provider + args: + - get-credentials + matchImages: + - "123456789.dkr.ecr.us-east-1.amazonaws.com" + defaultCacheDuration: "0s" + apiVersion: credentialprovider.kubelet.k8s.io/v1 + - name: static-credential-provider + args: + - /etc/kubernetes/static-image-credentials.json + matchImages: + - "anotherregistry.com" + defaultCacheDuration: "0s" + apiVersion: credentialprovider.kubelet.k8s.io/v1 +`, + }, + }, + { + name: "ECR global mirror image registry", + credentials: []providerConfig{ + { + URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com", + }, + { + URL: "https://98765432.dkr.ecr.us-east-1.amazonaws.com", + Mirror: true, + }, + }, + want: &cabpkv1.File{ + Path: "/etc/kubernetes/dynamic-credential-provider-config.yaml", + Owner: "", + Permissions: "0600", + Encoding: "", + Append: false, + Content: `apiVersion: credentialprovider.d2iq.com/v1alpha1 +kind: DynamicCredentialProviderConfig +mirror: + endpoint: 98765432.dkr.ecr.us-east-1.amazonaws.com + credentialsStrategy: MirrorCredentialsFirst +credentialProviderPluginBinDir: /etc/kubernetes/image-credential-provider/ +credentialProviders: + apiVersion: kubelet.config.k8s.io/v1 + kind: CredentialProviderConfig + providers: + - name: ecr-credential-provider + args: + - get-credentials + matchImages: + - "123456789.dkr.ecr.us-east-1.amazonaws.com" + defaultCacheDuration: "0s" + apiVersion: credentialprovider.kubelet.k8s.io/v1 + - name: ecr-credential-provider + args: + - get-credentials + matchImages: + - "98765432.dkr.ecr.us-east-1.amazonaws.com" + defaultCacheDuration: "0s" + apiVersion: credentialprovider.kubelet.k8s.io/v1 +`, + }, + }, + { + name: "Global mirror image registry with static credentials", + credentials: []providerConfig{ + { + URL: "https://myregistry.com", + Username: "myuser", + Password: "mypassword", + }, + { + URL: "https://mymirror.com", + Username: "mirroruser", + Password: "mirrorpassword", + Mirror: true, + }, + }, + want: &cabpkv1.File{ + Path: "/etc/kubernetes/dynamic-credential-provider-config.yaml", + Owner: "", + Permissions: "0600", + Encoding: "", + Append: false, + Content: `apiVersion: credentialprovider.d2iq.com/v1alpha1 +kind: DynamicCredentialProviderConfig +mirror: + endpoint: mymirror.com + credentialsStrategy: MirrorCredentialsFirst +credentialProviderPluginBinDir: /etc/kubernetes/image-credential-provider/ +credentialProviders: + apiVersion: kubelet.config.k8s.io/v1 + kind: CredentialProviderConfig + providers: + - name: static-credential-provider + args: + - /etc/kubernetes/static-image-credentials.json + matchImages: + - "myregistry.com" + defaultCacheDuration: "0s" + apiVersion: credentialprovider.kubelet.k8s.io/v1 + - name: static-credential-provider + args: + - /etc/kubernetes/static-image-credentials.json + matchImages: + - "mymirror.com" + defaultCacheDuration: "0s" + apiVersion: credentialprovider.kubelet.k8s.io/v1 +`, + }, + }, + } + for idx := range tests { + tt := tests[idx] + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + file, err := templateDynamicCredentialProviderConfig(tt.credentials) + require.ErrorIs(t, err, tt.wantErr) + assert.Equal(t, tt.want, file) + }) + } +} diff --git a/pkg/handlers/deleteinv0280/generic/mutation/imageregistries/credentials/credential_provider_install_files.go b/pkg/handlers/deleteinv0280/generic/mutation/imageregistries/credentials/credential_provider_install_files.go new file mode 100644 index 000000000..d024b94b9 --- /dev/null +++ b/pkg/handlers/deleteinv0280/generic/mutation/imageregistries/credentials/credential_provider_install_files.go @@ -0,0 +1,77 @@ +// Copyright 2023 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package credentials + +import ( + "bytes" + _ "embed" + "fmt" + "text/template" + + cabpkv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" + + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/common" +) + +var ( + installKubeletCredentialProvidersScriptOnRemote = common.ConfigFilePathOnRemote( + "install-kubelet-credential-providers.sh") + + installKubeletCredentialProvidersScriptOnRemoteCommand = "/bin/bash " + installKubeletCredentialProvidersScriptOnRemote +) + +const ( + //nolint:gosec // Does not contain hard coded credentials. + dynamicCredentialProviderImage = "ghcr.io/mesosphere/dynamic-credential-provider:v0.5.3" + + //nolint:gosec // Does not contain hard coded credentials. + credentialProviderTargetDir = "/etc/kubernetes/image-credential-provider/" +) + +var ( + //go:embed templates/install-kubelet-credential-providers.sh.gotmpl + installKubeletCredentialProvidersScript []byte + + installKubeletCredentialProvidersScriptTemplate = template.Must( + template.New("").Parse(string(installKubeletCredentialProvidersScript)), + ) +) + +func templateFilesAndCommandsForInstallKubeletCredentialProviders() ([]cabpkv1.File, []string, error) { + var files []cabpkv1.File + var commands []string + + installKCPScriptFile, installKCPScriptCommand, err := templateInstallKubeletCredentialProviders() + if err != nil { + return nil, nil, err + } + if installKCPScriptFile != nil { + files = append(files, *installKCPScriptFile) + commands = append(commands, installKCPScriptCommand) + } + + return files, commands, nil +} + +func templateInstallKubeletCredentialProviders() (*cabpkv1.File, string, error) { + templateInput := struct { + DynamicCredentialProviderImage string + CredentialProviderTargetDir string + }{ + DynamicCredentialProviderImage: dynamicCredentialProviderImage, + CredentialProviderTargetDir: credentialProviderTargetDir, + } + + var b bytes.Buffer + err := installKubeletCredentialProvidersScriptTemplate.Execute(&b, templateInput) + if err != nil { + return nil, "", fmt.Errorf("failed executing template: %w", err) + } + + return &cabpkv1.File{ + Path: installKubeletCredentialProvidersScriptOnRemote, + Content: b.String(), + Permissions: "0700", + }, installKubeletCredentialProvidersScriptOnRemoteCommand, nil +} diff --git a/pkg/handlers/deleteinv0280/generic/mutation/imageregistries/credentials/credential_provider_kubelet_args.go b/pkg/handlers/deleteinv0280/generic/mutation/imageregistries/credentials/credential_provider_kubelet_args.go new file mode 100644 index 000000000..33f6b7642 --- /dev/null +++ b/pkg/handlers/deleteinv0280/generic/mutation/imageregistries/credentials/credential_provider_kubelet_args.go @@ -0,0 +1,9 @@ +// Copyright 2023 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package credentials + +func addImageCredentialProviderArgs(args map[string]string) { + args["image-credential-provider-bin-dir"] = credentialProviderTargetDir + args["image-credential-provider-config"] = kubeletImageCredentialProviderConfigOnRemote +} diff --git a/pkg/handlers/deleteinv0280/generic/mutation/imageregistries/credentials/credentials_secret.go b/pkg/handlers/deleteinv0280/generic/mutation/imageregistries/credentials/credentials_secret.go new file mode 100644 index 000000000..6e8302335 --- /dev/null +++ b/pkg/handlers/deleteinv0280/generic/mutation/imageregistries/credentials/credentials_secret.go @@ -0,0 +1,153 @@ +// Copyright 2023 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package credentials + +import ( + "bytes" + _ "embed" + "fmt" + "net/url" + "strings" + "text/template" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + cabpkv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" + + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/utils" +) + +const ( + secretKeyForStaticCredentialProviderConfig = "static-credential-provider" //nolint:gosec // Not a credential. +) + +var ( + //go:embed templates/static-credential-provider.json.gotmpl + staticCredentialProviderConfigPatch []byte + + staticCredentialProviderConfigPatchTemplate = template.Must( + template.New("").Parse(string(staticCredentialProviderConfigPatch)), + ) +) + +func generateCredentialsSecretFile(configs []providerConfig, clusterName string) *cabpkv1.File { + if !configsRequireStaticCredentials(configs) { + return nil + } + return &cabpkv1.File{ + Path: kubeletStaticCredentialProviderCredentialsOnRemote, + ContentFrom: &cabpkv1.FileSource{ + Secret: cabpkv1.SecretFileSource{ + Name: credentialSecretName(clusterName), + Key: secretKeyForStaticCredentialProviderConfig, + }, + }, + Permissions: "0600", + } +} + +// generateCredentialsSecret generates a Secret containing the config for the image registry. +// The function needs the cluster name to add the required move and cluster name labels. +func generateCredentialsSecret( + configs []providerConfig, clusterName, namespace string, +) (*corev1.Secret, error) { + if !configsRequireStaticCredentials(configs) { + return nil, nil + } + + staticCredentialProviderSecretContents, err := kubeletStaticCredentialProviderSecretContents( + configs, + ) + if err != nil { + return nil, err + } + secretData := map[string]string{ + secretKeyForStaticCredentialProviderConfig: staticCredentialProviderSecretContents, + } + + return &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Secret", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: credentialSecretName(clusterName), + Namespace: namespace, + Labels: utils.NewLabels(utils.WithMove(), utils.WithClusterName(clusterName)), + }, + StringData: secretData, + Type: corev1.SecretTypeOpaque, + }, nil +} + +func kubeletStaticCredentialProviderSecretContents(configs []providerConfig) (string, error) { + type templateInput struct { + RegistryHost string + Username string + Password string + } + + var inputs []templateInput //nolint:prealloc // We don't know the size of the slice yet. + for _, config := range configs { + requiresStaticCredentials, err := config.requiresStaticCredentials() + if err != nil { + return "", fmt.Errorf( + "error determining if Image Registry is a supported provider: %w", + err, + ) + } + if !requiresStaticCredentials { + continue + } + + registryURL, err := url.ParseRequestURI(config.URL) + if err != nil { + return "", fmt.Errorf("failed parsing registry URL: %w", err) + } + + inputs = append(inputs, templateInput{ + RegistryHost: registryURL.Host, + Username: config.Username, + Password: config.Password, + }) + + // Preserve special handling of "registry-1.docker.io" and add "docker.io" as an alias. + if registryURL.Host == "registry-1.docker.io" { + inputs = append(inputs, templateInput{ + RegistryHost: "docker.io", + Username: config.Username, + Password: config.Password, + }) + } + } + + if len(inputs) == 0 { + return "", nil + } + + var b bytes.Buffer + err := staticCredentialProviderConfigPatchTemplate.Execute(&b, inputs) + if err != nil { + return "", fmt.Errorf("failed executing template: %w", err) + } + + return strings.TrimSpace(b.String()), nil +} + +func configsRequireStaticCredentials(configs []providerConfig) bool { + for _, config := range configs { + requiresStaticCredentials, err := config.requiresStaticCredentials() + if err != nil { + return false + } + if requiresStaticCredentials { + return true + } + } + return false +} + +func credentialSecretName(clusterName string) string { + return fmt.Sprintf("%s-static-credential-provider-response", clusterName) +} diff --git a/pkg/handlers/deleteinv0280/generic/mutation/imageregistries/credentials/credentials_secret_test.go b/pkg/handlers/deleteinv0280/generic/mutation/imageregistries/credentials/credentials_secret_test.go new file mode 100644 index 000000000..ac8aeb708 --- /dev/null +++ b/pkg/handlers/deleteinv0280/generic/mutation/imageregistries/credentials/credentials_secret_test.go @@ -0,0 +1,230 @@ +// Copyright 2024 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package credentials + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + cabpkv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" +) + +func Test_generateCredentialsSecretFile(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + configs []providerConfig + clusterName string + wantFile *cabpkv1.File + }{ + { + name: "empty configs, expect no file", + configs: nil, + clusterName: "test-cluster", + wantFile: nil, + }, + { + name: "config with no static credentials, expect no file", + configs: []providerConfig{ + {URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com"}, + }, + clusterName: "test-cluster", + wantFile: nil, + }, + { + name: "config with static credentials, expect a file", + configs: []providerConfig{ + { + URL: "https://myregistry.com", + Username: "myuser", + Password: "mypassword", + }, + }, + clusterName: "test-cluster", + wantFile: &cabpkv1.File{ + Path: "/etc/kubernetes/static-image-credentials.json", + Permissions: "0600", + ContentFrom: &cabpkv1.FileSource{ + Secret: cabpkv1.SecretFileSource{ + Name: "test-cluster-static-credential-provider-response", + Key: "static-credential-provider", + }, + }, + }, + }, + } + + for idx := range tests { + tt := tests[idx] + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + gotFile := generateCredentialsSecretFile(tt.configs, tt.clusterName) + assert.Equal(t, tt.wantFile, gotFile) + }) + } +} + +func Test_generateCredentialsSecret(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + configs []providerConfig + clusterName string + namespace string + wantSecret *corev1.Secret + }{ + { + name: "empty configs, expect no Secret", + configs: nil, + clusterName: "test-cluster", + namespace: "test-namespace", + wantSecret: nil, + }, + { + name: "config with no static credentials, expect no Secret", + configs: []providerConfig{ + {URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com"}, + }, + clusterName: "test-cluster", + namespace: "test-namespace", + wantSecret: nil, + }, + { + name: "config with static credentials, expect a Secret", + configs: []providerConfig{ + { + URL: "https://myregistry.com", + Username: "myuser", + Password: "mypassword", + }, + }, + clusterName: "test-cluster", + namespace: "test-namespace", + wantSecret: &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Secret", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster-static-credential-provider-response", + Namespace: "test-namespace", + Labels: map[string]string{ + "cluster.x-k8s.io/cluster-name": "test-cluster", + "clusterctl.cluster.x-k8s.io/move": "", + }, + }, + StringData: map[string]string{ + "static-credential-provider": `{ + "kind":"CredentialProviderResponse", + "apiVersion":"credentialprovider.kubelet.k8s.io/v1", + "cacheKeyType":"Image", + "cacheDuration":"0s", + "auth":{ + "myregistry.com": {"username": "myuser", "password": "mypassword"} + } +}`, + }, + Type: corev1.SecretTypeOpaque, + }, + }, + } + + for idx := range tests { + tt := tests[idx] + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + gotSecret, err := generateCredentialsSecret(tt.configs, tt.clusterName, tt.namespace) + require.NoError(t, err) + assert.Equal(t, tt.wantSecret, gotSecret) + }) + } +} + +func Test_kubeletStaticCredentialProviderSecretContents(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + configs []providerConfig + wantContents string + }{ + { + name: "config with no static credentials, expect empty string", + configs: []providerConfig{ + {URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com"}, + }, + wantContents: "", + }, + { + name: "config with 'registry-1.docker.io', expect it to also add 'docker.io'", + configs: []providerConfig{ + { + URL: "https://registry-1.docker.io", + Username: "myuser", + Password: "mypassword", + }, + }, + wantContents: `{ + "kind":"CredentialProviderResponse", + "apiVersion":"credentialprovider.kubelet.k8s.io/v1", + "cacheKeyType":"Image", + "cacheDuration":"0s", + "auth":{ + "registry-1.docker.io": {"username": "myuser", "password": "mypassword"}, + "docker.io": {"username": "myuser", "password": "mypassword"} + } +}`, + }, + { + name: "multiple configs with some static credentials, expect a string with multiple entries", + configs: []providerConfig{ + { + URL: "https://myregistry.com", + Username: "myuser", + Password: "mypassword", + }, + {URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com"}, + { + URL: "https://registry-1.docker.io", + Username: "myuser", + Password: "mypassword", + }, + { + URL: "https://anotherregistry.com", + Username: "anotheruser", + Password: "anotherpassword", + }, + }, + wantContents: `{ + "kind":"CredentialProviderResponse", + "apiVersion":"credentialprovider.kubelet.k8s.io/v1", + "cacheKeyType":"Image", + "cacheDuration":"0s", + "auth":{ + "myregistry.com": {"username": "myuser", "password": "mypassword"}, + "registry-1.docker.io": {"username": "myuser", "password": "mypassword"}, + "docker.io": {"username": "myuser", "password": "mypassword"}, + "anotherregistry.com": {"username": "anotheruser", "password": "anotherpassword"} + } +}`, + }, + } + for idx := range tests { + tt := tests[idx] + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + gotContents, err := kubeletStaticCredentialProviderSecretContents(tt.configs) + require.NoError(t, err) + assert.Equal(t, tt.wantContents, gotContents) + }) + } +} diff --git a/pkg/handlers/deleteinv0280/generic/mutation/imageregistries/credentials/doc.go b/pkg/handlers/deleteinv0280/generic/mutation/imageregistries/credentials/doc.go new file mode 100644 index 000000000..c810a8eae --- /dev/null +++ b/pkg/handlers/deleteinv0280/generic/mutation/imageregistries/credentials/doc.go @@ -0,0 +1,5 @@ +// Copyright 2023 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// +kubebuilder:rbac:groups="",resources=secrets,verbs=watch;list;get;patch;create;update +package credentials diff --git a/pkg/handlers/deleteinv0280/generic/mutation/imageregistries/credentials/inject.go b/pkg/handlers/deleteinv0280/generic/mutation/imageregistries/credentials/inject.go new file mode 100644 index 000000000..dcd59bda1 --- /dev/null +++ b/pkg/handlers/deleteinv0280/generic/mutation/imageregistries/credentials/inject.go @@ -0,0 +1,478 @@ +// Copyright 2023 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package credentials + +import ( + "context" + "errors" + "fmt" + + corev1 "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" + controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1" + runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" + ctrl "sigs.k8s.io/controller-runtime" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers/mutation" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/patches" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/patches/selectors" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/variables" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/k8s/client" + handlersutils "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/utils" +) + +type imageRegistriesPatchHandler struct { + client ctrlclient.Client + + variableName string + variableFieldPath []string +} + +var ErrCredentialsNotFound = errors.New("registry credentials not found") + +func NewPatch( + cl ctrlclient.Client, +) *imageRegistriesPatchHandler { + return newImageRegistriesPatchHandler( + cl, + v1alpha1.ClusterConfigVariableName, + v1alpha1.ImageRegistriesVariableName, + ) +} + +func newImageRegistriesPatchHandler( + cl ctrlclient.Client, + variableName string, + variableFieldPath ...string, +) *imageRegistriesPatchHandler { + scheme := runtime.NewScheme() + _ = bootstrapv1.AddToScheme(scheme) + _ = controlplanev1.AddToScheme(scheme) + return &imageRegistriesPatchHandler{ + client: cl, + variableName: variableName, + variableFieldPath: variableFieldPath, + } +} + +func (h *imageRegistriesPatchHandler) Mutate( + ctx context.Context, + obj *unstructured.Unstructured, + vars map[string]apiextensionsv1.JSON, + holderRef runtimehooksv1.HolderReference, + clusterKey ctrlclient.ObjectKey, + clusterGetter mutation.ClusterGetter, +) error { + log := ctrl.LoggerFrom(ctx).WithValues( + "holderRef", holderRef, + ) + + imageRegistries, imageRegistriesErr := variables.Get[[]v1alpha1.ImageRegistry]( + vars, + h.variableName, + h.variableFieldPath..., + ) + + // add credentials for global image registry mirror + globalMirror, globalMirrorErr := variables.Get[v1alpha1.GlobalImageRegistryMirror]( + vars, + h.variableName, + v1alpha1.GlobalMirrorVariableName, + ) + + switch { + case variables.IsNotFoundError(imageRegistriesErr) && variables.IsNotFoundError(globalMirrorErr): + log.V(5).Info("Image Registry Credentials and Global Registry Mirror variable not defined") + return nil + case imageRegistriesErr != nil && !variables.IsNotFoundError(imageRegistriesErr): + return imageRegistriesErr + case globalMirrorErr != nil && !variables.IsNotFoundError(globalMirrorErr): + return globalMirrorErr + } + + registriesWithOptionalCredentials := make([]providerConfig, 0, len(imageRegistries)) + for _, imageRegistry := range imageRegistries { + registryWithOptionalCredentials, generateErr := registryWithOptionalCredentialsFromImageRegistryCredentials( + ctx, + h.client, + imageRegistry, + obj, + ) + if generateErr != nil { + return generateErr + } + + registriesWithOptionalCredentials = append( + registriesWithOptionalCredentials, + registryWithOptionalCredentials, + ) + } + + if globalMirrorErr == nil { + mirrorCredentials, generateErr := mirrorWithOptionalCredentialsFromGlobalImageRegistryMirror( + ctx, + h.client, + globalMirror, + obj, + ) + if generateErr != nil { + return generateErr + } + registriesWithOptionalCredentials = append( + registriesWithOptionalCredentials, + mirrorCredentials, + ) + } + + registriesThatNeedConfiguration, err := providerConfigsThatNeedConfiguration( + registriesWithOptionalCredentials, + ) + if err != nil { + return err + } + if len(registriesThatNeedConfiguration) == 0 { + log.V(5).Info("Image registry credentials are not needed") + return nil + } + + files, commands, generateErr := generateFilesAndCommands( + registriesThatNeedConfiguration, + clusterKey.Name, + ) + if generateErr != nil { + return generateErr + } + + if err := patches.MutateIfApplicable( + obj, vars, &holderRef, selectors.ControlPlane(), log, + func(obj *controlplanev1.KubeadmControlPlaneTemplate) error { + log.WithValues( + "patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(), + "patchedObjectName", ctrlclient.ObjectKeyFromObject(obj), + ).Info("adding files to control plane kubeadm config spec") + obj.Spec.Template.Spec.KubeadmConfigSpec.Files = append( + obj.Spec.Template.Spec.KubeadmConfigSpec.Files, + files..., + ) + + log.WithValues( + "patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(), + "patchedObjectName", ctrlclient.ObjectKeyFromObject(obj), + ).Info("adding PreKubeadmCommands to control plane kubeadm config spec") + obj.Spec.Template.Spec.KubeadmConfigSpec.PreKubeadmCommands = append( + obj.Spec.Template.Spec.KubeadmConfigSpec.PreKubeadmCommands, + commands..., + ) + + cluster, err := clusterGetter(ctx) + if err != nil { + log.Error( + err, + "failed to get cluster from Image Registry Credentials mutation handler", + ) + return err + } + + err = ensureOwnerReferenceOnCredentialsSecrets(ctx, h.client, imageRegistries, globalMirror, cluster) + if err != nil { + return err + } + + err = createSecretIfNeeded(ctx, h.client, registriesThatNeedConfiguration, cluster) + if err != nil { + return err + } + + initConfiguration := obj.Spec.Template.Spec.KubeadmConfigSpec.InitConfiguration + if initConfiguration == nil { + initConfiguration = &bootstrapv1.InitConfiguration{} + } + obj.Spec.Template.Spec.KubeadmConfigSpec.InitConfiguration = initConfiguration + if initConfiguration.NodeRegistration.KubeletExtraArgs == nil { + initConfiguration.NodeRegistration.KubeletExtraArgs = map[string]string{} + } + addImageCredentialProviderArgs(initConfiguration.NodeRegistration.KubeletExtraArgs) + + joinConfiguration := obj.Spec.Template.Spec.KubeadmConfigSpec.JoinConfiguration + if joinConfiguration == nil { + joinConfiguration = &bootstrapv1.JoinConfiguration{} + } + obj.Spec.Template.Spec.KubeadmConfigSpec.JoinConfiguration = joinConfiguration + if joinConfiguration.NodeRegistration.KubeletExtraArgs == nil { + joinConfiguration.NodeRegistration.KubeletExtraArgs = map[string]string{} + } + addImageCredentialProviderArgs(joinConfiguration.NodeRegistration.KubeletExtraArgs) + return nil + }); err != nil { + return err + } + + if err := patches.MutateIfApplicable( + obj, vars, &holderRef, selectors.WorkersKubeadmConfigTemplateSelector(), log, + func(obj *bootstrapv1.KubeadmConfigTemplate) error { + log.WithValues( + "patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(), + "patchedObjectName", ctrlclient.ObjectKeyFromObject(obj), + ).Info("adding files to worker node kubeadm config template") + obj.Spec.Template.Spec.Files = append(obj.Spec.Template.Spec.Files, files...) + + log.WithValues( + "patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(), + "patchedObjectName", ctrlclient.ObjectKeyFromObject(obj), + ).Info("adding PreKubeadmCommands to worker node kubeadm config template") + obj.Spec.Template.Spec.PreKubeadmCommands = append(obj.Spec.Template.Spec.PreKubeadmCommands, commands...) + + cluster, err := clusterGetter(ctx) + if err != nil { + log.Error( + err, + "failed to get cluster from Image Registry Credentials mutation handler", + ) + return err + } + + err = ensureOwnerReferenceOnCredentialsSecrets(ctx, h.client, imageRegistries, globalMirror, cluster) + if err != nil { + return err + } + + err = createSecretIfNeeded(ctx, h.client, registriesThatNeedConfiguration, cluster) + if err != nil { + return err + } + + joinConfiguration := obj.Spec.Template.Spec.JoinConfiguration + if joinConfiguration == nil { + joinConfiguration = &bootstrapv1.JoinConfiguration{} + } + obj.Spec.Template.Spec.JoinConfiguration = joinConfiguration + if joinConfiguration.NodeRegistration.KubeletExtraArgs == nil { + joinConfiguration.NodeRegistration.KubeletExtraArgs = map[string]string{} + } + addImageCredentialProviderArgs(joinConfiguration.NodeRegistration.KubeletExtraArgs) + + return nil + }); err != nil { + return err + } + + return nil +} + +func ensureOwnerReferenceOnCredentialsSecrets( + ctx context.Context, + c ctrlclient.Client, + imageRegistries []v1alpha1.ImageRegistry, + globalMirror v1alpha1.GlobalImageRegistryMirror, + cluster *clusterv1.Cluster, +) error { + var credentials []*v1alpha1.RegistryCredentials + for _, imageRegistry := range imageRegistries { + if imageRegistry.Credentials != nil { + credentials = append(credentials, imageRegistry.Credentials) + } + } + if globalMirror.Credentials != nil { + credentials = append(credentials, globalMirror.Credentials) + } + + for _, credential := range credentials { + if secretName := handlersutils.SecretNameForImageRegistryCredentials(credential); secretName != "" { + // Ensure the Secret is owned by the Cluster so it is correctly moved and deleted with the Cluster. + // This code assumes that Secret exists and that was validated before calling this function. + err := handlersutils.EnsureClusterOwnerReferenceForObject( + ctx, + c, + corev1.TypedLocalObjectReference{ + Kind: "Secret", + Name: secretName, + }, + cluster, + ) + if err != nil { + return fmt.Errorf( + "error updating owner references on image registry Secret: %w", + err, + ) + } + } + } + + return nil +} + +func registryWithOptionalCredentialsFromImageRegistryCredentials( + ctx context.Context, + c ctrlclient.Client, + imageRegistry v1alpha1.ImageRegistry, + obj ctrlclient.Object, +) (providerConfig, error) { + registryWithOptionalCredentials := providerConfig{ + URL: imageRegistry.URL, + } + secret, err := handlersutils.SecretForImageRegistryCredentials( + ctx, + c, + imageRegistry.Credentials, + obj.GetNamespace(), + ) + if err != nil { + return providerConfig{}, fmt.Errorf( + "error getting secret %s/%s from Image Registry variable: %w", + obj.GetNamespace(), + imageRegistry.Credentials.SecretRef.Name, + err, + ) + } + + if secret != nil { + registryWithOptionalCredentials.Username = string(secret.Data["username"]) + registryWithOptionalCredentials.Password = string(secret.Data["password"]) + registryWithOptionalCredentials.HasCACert = secretHasCACert(secret) + } + + return registryWithOptionalCredentials, nil +} + +func mirrorWithOptionalCredentialsFromGlobalImageRegistryMirror( + ctx context.Context, + c ctrlclient.Client, + mirror v1alpha1.GlobalImageRegistryMirror, + obj ctrlclient.Object, +) (providerConfig, error) { + mirrorCredentials := providerConfig{ + URL: mirror.URL, + Mirror: true, + } + secret, err := handlersutils.SecretForImageRegistryCredentials( + ctx, + c, + mirror.Credentials, + obj.GetNamespace(), + ) + if err != nil { + return providerConfig{}, fmt.Errorf( + "error getting secret %s/%s from Global Image Registry Mirror variable: %w", + obj.GetNamespace(), + mirror.Credentials.SecretRef.Name, + err, + ) + } + + if secret != nil { + mirrorCredentials.Username = string(secret.Data["username"]) + mirrorCredentials.Password = string(secret.Data["password"]) + mirrorCredentials.HasCACert = secretHasCACert(secret) + } + + return mirrorCredentials, nil +} + +func generateFilesAndCommands( + registriesWithOptionalCredentials []providerConfig, + clusterName string, +) ([]bootstrapv1.File, []string, error) { + files, commands, err := templateFilesAndCommandsForInstallKubeletCredentialProviders() + if err != nil { + return nil, nil, fmt.Errorf( + "error generating install files and commands for Image Registry Credentials variable: %w", + err, + ) + } + imageCredentialProviderConfigFiles, err := templateFilesForImageCredentialProviderConfigs( + registriesWithOptionalCredentials, + ) + if err != nil { + return nil, nil, fmt.Errorf( + "error generating files for Image Registry Credentials variable: %w", + err, + ) + } + files = append(files, imageCredentialProviderConfigFiles...) + + credentialSecretFile := generateCredentialsSecretFile( + registriesWithOptionalCredentials, + clusterName, + ) + if credentialSecretFile != nil { + files = append(files, *credentialSecretFile) + } + + return files, commands, err +} + +func createSecretIfNeeded( + ctx context.Context, + c ctrlclient.Client, + registriesWithOptionalCredentials []providerConfig, + cluster *clusterv1.Cluster, +) error { + credentialsSecret, err := generateCredentialsSecret( + registriesWithOptionalCredentials, + cluster.Name, + cluster.Namespace, + ) + if err != nil { + return fmt.Errorf( + "error generating credentials Secret for Image Registry Credentials variable: %w", + err, + ) + } + if credentialsSecret != nil { + if err = controllerutil.SetOwnerReference(cluster, credentialsSecret, c.Scheme()); err != nil { + return fmt.Errorf( + "failed to set owner reference on Image Registry Credentials Secret: %w", + err, + ) + } + if err := client.ServerSideApply(ctx, c, credentialsSecret, client.ForceOwnership); err != nil { + return fmt.Errorf("failed to apply Image Registry Credentials Secret: %w", err) + } + } + + return nil +} + +// This handler reads input from two user provided variables: globalImageRegistryMirror and imageRegistries. +// We expect if imageRegistries is set it will either have static credentials +// or be for a registry where the credential plugin returns the credentials, ie ECR, GCR, ACR, etc, +// or have no credentials set but to contain a CA cert, +// and if that is not the case we assume the users missed setting static credentials and return an error. +// However, in addition to passing credentials with the globalImageRegistryMirror variable, +// it can also be used to only set Containerd mirror configuration, +// in which case it is valid for static credentials to not be set and will be skipped, no error +// and this handler will skip generating any credential plugin related configuration. +func providerConfigsThatNeedConfiguration(configs []providerConfig) ([]providerConfig, error) { + var needConfiguration []providerConfig //nolint:prealloc // We don't know the size of the slice yet. + for _, config := range configs { + requiresStaticCredentials, err := config.requiresStaticCredentials() + if err != nil { + return nil, + fmt.Errorf("error determining if Image Registry is a supported provider: %w", err) + } + // verify the credentials are actually set if the plugin requires static credentials + if config.isCredentialsEmpty() && requiresStaticCredentials { + if config.Mirror || config.HasCACert { + // not setting credentials for a mirror is valid, but won't need any configuration + // not setting credentials for a registry with a CA cert is valid, but won't need any configuration + continue + } + return nil, fmt.Errorf( + "invalid image registry: %s: %w", + config.URL, + ErrCredentialsNotFound, + ) + } + needConfiguration = append(needConfiguration, config) + } + + return needConfiguration, nil +} diff --git a/pkg/handlers/deleteinv0280/generic/mutation/imageregistries/credentials/inject_test.go b/pkg/handlers/deleteinv0280/generic/mutation/imageregistries/credentials/inject_test.go new file mode 100644 index 000000000..dad4e71ab --- /dev/null +++ b/pkg/handlers/deleteinv0280/generic/mutation/imageregistries/credentials/inject_test.go @@ -0,0 +1,570 @@ +// Copyright 2023 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package credentials + +import ( + "context" + "testing" + + . "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apiserver/pkg/storage/names" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" + "sigs.k8s.io/cluster-api/util" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers/mutation" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/testutils/capitest" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/testutils/capitest/request" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/test/helpers" +) + +const ( + validSecretName = "myregistry-credentials" +) + +func Test_providerConfigsThatNeedConfiguration(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + configs []providerConfig + expected []providerConfig + wantErr error + }{ + { + name: "ECR registry with no credentials", + configs: []providerConfig{ + {URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com"}, + }, + expected: []providerConfig{ + {URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com"}, + }, + }, + { + name: "registry with static credentials", + configs: []providerConfig{{ + URL: "https://myregistry.com", + Username: "myuser", + Password: "mypassword", + }}, + expected: []providerConfig{{ + URL: "https://myregistry.com", + Username: "myuser", + Password: "mypassword", + }}, + }, + { + name: "ECR mirror with no credentials", + configs: []providerConfig{{ + URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com", + Mirror: true, + }}, + expected: []providerConfig{{ + URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com", + Mirror: true, + }}, + }, + { + name: "mirror with static credentials", + configs: []providerConfig{{ + URL: "https://mymirror.com", + Username: "myuser", + Password: "mypassword", + Mirror: true, + }}, + expected: []providerConfig{{ + URL: "https://mymirror.com", + Username: "myuser", + Password: "mypassword", + Mirror: true, + }}, + }, + { + name: "mirror with no credentials", + configs: []providerConfig{{ + URL: "https://mymirror.com", + Mirror: true, + }}, + expected: nil, + }, + { + name: "a registry with static credentials and a mirror with no credentials", + configs: []providerConfig{ + { + URL: "https://myregistry.com", + Username: "myuser", + Password: "mypassword", + Mirror: false, + }, + { + URL: "https://mymirror.com", + Mirror: true, + }, + }, + expected: []providerConfig{ + { + URL: "https://myregistry.com", + Username: "myuser", + Password: "mypassword", + Mirror: false, + }, + }, + }, + { + name: "a registry with missing credentials and a mirror with no credentials", + configs: []providerConfig{ + { + URL: "https://myregistry.com", + Mirror: false, + }, + { + URL: "https://mymirror.com", + Mirror: true, + }, + }, + wantErr: ErrCredentialsNotFound, + }, + { + name: "registry with missing credentials", + configs: []providerConfig{{ + URL: "https://myregistry.com", + }}, + wantErr: ErrCredentialsNotFound, + }, + { + name: "registry with missing credentials but with a CA", + configs: []providerConfig{{ + URL: "https://myregistry.com", + HasCACert: true, + }}, + }, + { + name: "mirror with missing credentials but with a CA", + configs: []providerConfig{{ + URL: "https://mymirror.com", + HasCACert: true, + Mirror: true, + }}, + }, + } + + for idx := range testCases { + tt := testCases[idx] + + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + expected, err := providerConfigsThatNeedConfiguration(tt.configs) + require.ErrorIs(t, err, tt.wantErr) + assert.Equal(t, tt.expected, expected) + }) + } +} + +func TestImageRegistriesPatch(t *testing.T) { + gomega.RegisterFailHandler(Fail) + RunSpecs(t, "Image registry mutator suite") +} + +var _ = Describe("Generate Image registry patches", func() { + clientScheme := runtime.NewScheme() + utilruntime.Must(clientgoscheme.AddToScheme(clientScheme)) + utilruntime.Must(clusterv1.AddToScheme(clientScheme)) + + patchGenerator := func() mutation.GeneratePatches { + // Use direct client to allow patch handler to read objects created by tests. + client, err := helpers.TestEnv.GetK8sClientWithScheme(clientScheme) + gomega.Expect(err).To(gomega.BeNil()) + return mutation.NewMetaGeneratePatchesHandler("", client, NewPatch(client)).(mutation.GeneratePatches) + } + + testDefs := []struct { + capitest.PatchTestDef + expectOwnerReferenceOnSecrets bool + }{ + { + PatchTestDef: capitest.PatchTestDef{ + Name: "unset variable", + }, + }, + { + PatchTestDef: capitest.PatchTestDef{ + Name: "files added in KubeadmControlPlaneTemplate for ECR without a Secret", + Vars: []runtimehooksv1.Variable{ + capitest.VariableWithValue( + v1alpha1.ClusterConfigVariableName, + []v1alpha1.ImageRegistry{{ + URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com", + }}, + v1alpha1.ImageRegistriesVariableName, + ), + }, + RequestItem: request.NewKubeadmControlPlaneTemplateRequestItem(""), + ExpectedPatchMatchers: []capitest.JSONPatchMatcher{ + { + Operation: "add", + Path: "/spec/template/spec/kubeadmConfigSpec/files", + ValueMatcher: gomega.ContainElements( + gomega.HaveKeyWithValue( + "path", "/etc/caren/install-kubelet-credential-providers.sh", + ), + gomega.HaveKeyWithValue( + "path", "/etc/kubernetes/image-credential-provider-config.yaml", + ), + gomega.HaveKeyWithValue( + "path", "/etc/kubernetes/dynamic-credential-provider-config.yaml", + ), + ), + }, + { + Operation: "add", + Path: "/spec/template/spec/kubeadmConfigSpec/preKubeadmCommands", + ValueMatcher: gomega.ContainElement( + "/bin/bash /etc/caren/install-kubelet-credential-providers.sh", + ), + }, + { + Operation: "add", + Path: "/spec/template/spec/kubeadmConfigSpec/initConfiguration/nodeRegistration/kubeletExtraArgs", + ValueMatcher: gomega.HaveKeyWithValue( + "image-credential-provider-bin-dir", + "/etc/kubernetes/image-credential-provider/", + ), + }, + { + Operation: "add", + Path: "/spec/template/spec/kubeadmConfigSpec/joinConfiguration/nodeRegistration/kubeletExtraArgs", + ValueMatcher: gomega.HaveKeyWithValue( + "image-credential-provider-config", + "/etc/kubernetes/image-credential-provider-config.yaml", + ), + }, + }, + }, + }, + { + PatchTestDef: capitest.PatchTestDef{ + Name: "files added in KubeadmControlPlaneTemplate for registry with a Secret", + Vars: []runtimehooksv1.Variable{ + capitest.VariableWithValue( + v1alpha1.ClusterConfigVariableName, + []v1alpha1.ImageRegistry{{ + URL: "https://registry.example.com", + Credentials: &v1alpha1.RegistryCredentials{ + SecretRef: &v1alpha1.LocalObjectReference{ + Name: validSecretName, + }, + }, + }}, + v1alpha1.ImageRegistriesVariableName, + ), + }, + RequestItem: request.NewKubeadmControlPlaneTemplateRequest( + "", + "test-kubeadmconfigtemplate", + ), + ExpectedPatchMatchers: []capitest.JSONPatchMatcher{ + { + Operation: "add", + Path: "/spec/template/spec/kubeadmConfigSpec/files", + ValueMatcher: gomega.ContainElements( + gomega.HaveKeyWithValue( + "path", "/etc/caren/install-kubelet-credential-providers.sh", + ), + gomega.HaveKeyWithValue( + "path", "/etc/kubernetes/image-credential-provider-config.yaml", + ), + gomega.HaveKeyWithValue( + "path", "/etc/kubernetes/dynamic-credential-provider-config.yaml", + ), + gomega.HaveKeyWithValue( + "path", "/etc/kubernetes/static-image-credentials.json", + ), + ), + }, + { + Operation: "add", + Path: "/spec/template/spec/kubeadmConfigSpec/preKubeadmCommands", + ValueMatcher: gomega.ContainElement( + "/bin/bash /etc/caren/install-kubelet-credential-providers.sh", + ), + }, + { + Operation: "add", + Path: "/spec/template/spec/kubeadmConfigSpec/initConfiguration/nodeRegistration/kubeletExtraArgs", + ValueMatcher: gomega.HaveKeyWithValue( + "image-credential-provider-bin-dir", + "/etc/kubernetes/image-credential-provider/", + ), + }, + { + Operation: "add", + Path: "/spec/template/spec/kubeadmConfigSpec/joinConfiguration/nodeRegistration/kubeletExtraArgs", + ValueMatcher: gomega.HaveKeyWithValue( + "image-credential-provider-config", + "/etc/kubernetes/image-credential-provider-config.yaml", + ), + }, + }, + }, + expectOwnerReferenceOnSecrets: true, + }, + { + PatchTestDef: capitest.PatchTestDef{ + Name: "files added in KubeadmConfigTemplate for ECR without a Secret", + Vars: []runtimehooksv1.Variable{ + capitest.VariableWithValue( + v1alpha1.ClusterConfigVariableName, + []v1alpha1.ImageRegistry{{ + URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com", + }}, + v1alpha1.ImageRegistriesVariableName, + ), + capitest.VariableWithValue( + "builtin", + map[string]any{ + "machineDeployment": map[string]any{ + "class": names.SimpleNameGenerator.GenerateName("worker-"), + }, + }, + ), + }, + RequestItem: request.NewKubeadmConfigTemplateRequestItem(""), + ExpectedPatchMatchers: []capitest.JSONPatchMatcher{ + { + Operation: "add", + Path: "/spec/template/spec/files", + ValueMatcher: gomega.ContainElements( + gomega.HaveKeyWithValue( + "path", "/etc/caren/install-kubelet-credential-providers.sh", + ), + gomega.HaveKeyWithValue( + "path", "/etc/kubernetes/image-credential-provider-config.yaml", + ), + gomega.HaveKeyWithValue( + "path", "/etc/kubernetes/dynamic-credential-provider-config.yaml", + ), + ), + }, + { + Operation: "add", + Path: "/spec/template/spec/preKubeadmCommands", + ValueMatcher: gomega.ContainElement( + "/bin/bash /etc/caren/install-kubelet-credential-providers.sh", + ), + }, + { + Operation: "add", + Path: "/spec/template/spec/joinConfiguration/nodeRegistration/kubeletExtraArgs", + ValueMatcher: gomega.HaveKeyWithValue( + "image-credential-provider-bin-dir", + "/etc/kubernetes/image-credential-provider/", + ), + }, + }, + }, + }, + { + PatchTestDef: capitest.PatchTestDef{ + Name: "files added in KubeadmConfigTemplate for registry with a Secret", + Vars: []runtimehooksv1.Variable{ + capitest.VariableWithValue( + v1alpha1.ClusterConfigVariableName, + []v1alpha1.ImageRegistry{{ + URL: "https://registry.example.com", + Credentials: &v1alpha1.RegistryCredentials{ + SecretRef: &v1alpha1.LocalObjectReference{ + Name: validSecretName, + }, + }, + }}, + v1alpha1.ImageRegistriesVariableName, + ), + capitest.VariableWithValue( + "builtin", + map[string]any{ + "machineDeployment": map[string]any{ + "class": names.SimpleNameGenerator.GenerateName("worker-"), + }, + }, + ), + }, + RequestItem: request.NewKubeadmConfigTemplateRequest( + "", + "test-kubeadmconfigtemplate", + ), + ExpectedPatchMatchers: []capitest.JSONPatchMatcher{ + { + Operation: "add", + Path: "/spec/template/spec/files", + ValueMatcher: gomega.ContainElements( + gomega.HaveKeyWithValue( + "path", "/etc/caren/install-kubelet-credential-providers.sh", + ), + gomega.HaveKeyWithValue( + "path", "/etc/kubernetes/image-credential-provider-config.yaml", + ), + gomega.HaveKeyWithValue( + "path", "/etc/kubernetes/dynamic-credential-provider-config.yaml", + ), + gomega.HaveKeyWithValue( + "path", "/etc/kubernetes/static-image-credentials.json", + ), + ), + }, + { + Operation: "add", + Path: "/spec/template/spec/preKubeadmCommands", + ValueMatcher: gomega.ContainElement( + "/bin/bash /etc/caren/install-kubelet-credential-providers.sh", + ), + }, + { + Operation: "add", + Path: "/spec/template/spec/joinConfiguration/nodeRegistration/kubeletExtraArgs", + ValueMatcher: gomega.HaveKeyWithValue( + "image-credential-provider-bin-dir", + "/etc/kubernetes/image-credential-provider/", + ), + }, + }, + }, + expectOwnerReferenceOnSecrets: true, + }, + { + PatchTestDef: capitest.PatchTestDef{ + Name: "error for a registry with no credentials", + Vars: []runtimehooksv1.Variable{ + capitest.VariableWithValue( + v1alpha1.ClusterConfigVariableName, + []v1alpha1.ImageRegistry{{ + URL: "https://registry.example.com", + }}, + v1alpha1.ImageRegistriesVariableName, + ), + }, + RequestItem: request.NewKubeadmControlPlaneTemplateRequestItem(""), + ExpectedFailure: true, + }, + }, + } + + // Create credentials secret before each test + BeforeEach(func(ctx SpecContext) { + client, err := helpers.TestEnv.GetK8sClientWithScheme(clientScheme) + gomega.Expect(err).To(gomega.BeNil()) + + gomega.Expect(client.Create( + ctx, + newRegistryCredentialsSecret(validSecretName, request.Namespace), + )).To(gomega.BeNil()) + + gomega.Expect(client.Create( + ctx, + &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: request.ClusterName, + Namespace: request.Namespace, + }, + }, + )).To(gomega.BeNil()) + }) + + // Delete credentials secret after each test + AfterEach(func(ctx SpecContext) { + client, err := helpers.TestEnv.GetK8sClientWithScheme(clientScheme) + gomega.Expect(err).To(gomega.BeNil()) + + gomega.Expect(client.Delete( + ctx, + newRegistryCredentialsSecret(validSecretName, request.Namespace), + )).To(gomega.BeNil()) + + gomega.Expect(client.Delete( + ctx, + &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: request.ClusterName, + Namespace: request.Namespace, + }, + }, + )).To(gomega.BeNil()) + }) + // create test node for each case + for testIdx := range testDefs { + tt := testDefs[testIdx] + It(tt.Name, func() { + capitest.AssertGeneratePatches(GinkgoT(), patchGenerator, &tt.PatchTestDef) + + // validate an OwnerReference was added to the user provided and generated Secrets + if tt.expectOwnerReferenceOnSecrets { + client, err := helpers.TestEnv.GetK8sClientWithScheme(clientScheme) + gomega.Expect(err).To(gomega.BeNil()) + + // get the Cluster to use for the owner reference assertion + clusterKey := ctrlclient.ObjectKey{ + Namespace: request.Namespace, + Name: request.ClusterName, + } + cluster := &clusterv1.Cluster{} + gomega.Expect(client.Get( + context.Background(), + clusterKey, + cluster, + )).To(gomega.BeNil()) + for _, name := range []string{validSecretName, credentialSecretName(request.ClusterName)} { + key := ctrlclient.ObjectKey{ + Namespace: request.Namespace, + Name: name, + } + secret := &corev1.Secret{} + gomega.Expect(client.Get( + context.Background(), + key, + secret, + )).To(gomega.BeNil()) + + // assert the owner reference with the Cluster was added to the Secret + gomega.Expect(secret.OwnerReferences).ToNot(gomega.BeEmpty()) + ownerRef := metav1.OwnerReference{ + APIVersion: clusterv1.GroupVersion.String(), + Kind: cluster.Kind, + UID: cluster.UID, + Name: cluster.Name, + } + util.HasOwnerRef(secret.OwnerReferences, ownerRef) + } + } + }) + } +}) + +func newRegistryCredentialsSecret(name, namespace string) *corev1.Secret { + secretData := map[string][]byte{ + "username": []byte("myuser"), + "password": []byte("mypassword"), + } + return &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Secret", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Data: secretData, + Type: corev1.SecretTypeOpaque, + } +} diff --git a/pkg/handlers/deleteinv0280/generic/mutation/imageregistries/credentials/templates/dynamic-credential-provider-config.yaml.gotmpl b/pkg/handlers/deleteinv0280/generic/mutation/imageregistries/credentials/templates/dynamic-credential-provider-config.yaml.gotmpl new file mode 100644 index 000000000..6bbf3fd95 --- /dev/null +++ b/pkg/handlers/deleteinv0280/generic/mutation/imageregistries/credentials/templates/dynamic-credential-provider-config.yaml.gotmpl @@ -0,0 +1,33 @@ +apiVersion: credentialprovider.d2iq.com/v1alpha1 +kind: DynamicCredentialProviderConfig +{{- range .}} +{{- if .Mirror }} +mirror: + endpoint: {{ .RegistryHost }} + credentialsStrategy: MirrorCredentialsFirst +{{- break }} +{{- end }} +{{- end }} +credentialProviderPluginBinDir: /etc/kubernetes/image-credential-provider/ +credentialProviders: + apiVersion: kubelet.config.k8s.io/v1 + kind: CredentialProviderConfig + providers: + {{- range . }} + - name: {{ .ProviderBinary }} + {{- with .ProviderArgs }} + args: + {{- range . }} + - {{ . }} + {{- end }} + {{- end }} + matchImages: + {{- with .RegistryHost }} + - {{ printf "%q" . }} + {{- if eq . "registry-1.docker.io" }} + - "docker.io" + {{- end }} + {{- end }} + defaultCacheDuration: "0s" + apiVersion: {{ .ProviderAPIVersion }} + {{- end }} diff --git a/pkg/handlers/deleteinv0280/generic/mutation/imageregistries/credentials/templates/install-kubelet-credential-providers.sh.gotmpl b/pkg/handlers/deleteinv0280/generic/mutation/imageregistries/credentials/templates/install-kubelet-credential-providers.sh.gotmpl new file mode 100644 index 000000000..30eb73517 --- /dev/null +++ b/pkg/handlers/deleteinv0280/generic/mutation/imageregistries/credentials/templates/install-kubelet-credential-providers.sh.gotmpl @@ -0,0 +1,23 @@ +#!/bin/bash +set -euo pipefail +IFS=$'\n\t' + +declare -r CREDENTIAL_PROVIDER_IMAGE="{{ .DynamicCredentialProviderImage }}" + +if ! ctr --namespace k8s.io images check "name==${CREDENTIAL_PROVIDER_IMAGE}" | grep "${CREDENTIAL_PROVIDER_IMAGE}" >/dev/null; then + ctr --namespace k8s.io images pull "${CREDENTIAL_PROVIDER_IMAGE}" +fi + +cleanup() { + ctr images unmount "${tmp_ctr_mount_dir}" || true +} + +trap 'cleanup' EXIT + +readonly tmp_ctr_mount_dir="$(mktemp -d)" + +export CREDENTIAL_PROVIDER_SOURCE_DIR="${tmp_ctr_mount_dir}/opt/image-credential-provider/bin/" +export CREDENTIAL_PROVIDER_TARGET_DIR="{{ .CredentialProviderTargetDir }}" + +ctr --namespace k8s.io images mount "${CREDENTIAL_PROVIDER_IMAGE}" "${tmp_ctr_mount_dir}" +"${tmp_ctr_mount_dir}/opt/image-credential-provider/bin/dynamic-credential-provider" install diff --git a/pkg/handlers/deleteinv0280/generic/mutation/imageregistries/credentials/templates/kubelet-image-credential-provider-config.yaml.gotmpl b/pkg/handlers/deleteinv0280/generic/mutation/imageregistries/credentials/templates/kubelet-image-credential-provider-config.yaml.gotmpl new file mode 100644 index 000000000..674e84dc6 --- /dev/null +++ b/pkg/handlers/deleteinv0280/generic/mutation/imageregistries/credentials/templates/kubelet-image-credential-provider-config.yaml.gotmpl @@ -0,0 +1,27 @@ +apiVersion: kubelet.config.k8s.io/v1 +kind: CredentialProviderConfig +providers: +- name: {{ .ProviderBinary }} + {{- with .ProviderArgs }} + args: + {{- range . }} + - {{ . }} + {{- end }} + {{- end }} + matchImages: + {{- range .RegistryHosts}} + {{- with . }} + - {{ printf "%q" . }} + {{- if eq . "registry-1.docker.io" }} + - "docker.io" + {{- end }} + {{- end }} + {{- end }} + - "*" + - "*.*" + - "*.*.*" + - "*.*.*.*" + - "*.*.*.*.*" + - "*.*.*.*.*.*" + defaultCacheDuration: "0s" + apiVersion: {{ .ProviderAPIVersion }} diff --git a/pkg/handlers/deleteinv0280/generic/mutation/imageregistries/credentials/templates/static-credential-provider.json.gotmpl b/pkg/handlers/deleteinv0280/generic/mutation/imageregistries/credentials/templates/static-credential-provider.json.gotmpl new file mode 100644 index 000000000..600fecf05 --- /dev/null +++ b/pkg/handlers/deleteinv0280/generic/mutation/imageregistries/credentials/templates/static-credential-provider.json.gotmpl @@ -0,0 +1,11 @@ +{ + "kind":"CredentialProviderResponse", + "apiVersion":"credentialprovider.kubelet.k8s.io/v1", + "cacheKeyType":"Image", + "cacheDuration":"0s", + "auth":{ + {{- range $i, $config := . }}{{ if $i }},{{ end}} + {{ printf "%q" $config.RegistryHost }}: {"username": {{ printf "%q" $config.Username }}, "password": {{ printf "%q" $config.Password }}} + {{- end }} + } +} diff --git a/pkg/handlers/deleteinv0280/generic/mutation/imageregistries/credentials/variables_test.go b/pkg/handlers/deleteinv0280/generic/mutation/imageregistries/credentials/variables_test.go new file mode 100644 index 000000000..6942b0ac6 --- /dev/null +++ b/pkg/handlers/deleteinv0280/generic/mutation/imageregistries/credentials/variables_test.go @@ -0,0 +1,108 @@ +// Copyright 2023 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package credentials + +import ( + "testing" + + "k8s.io/utils/ptr" + + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/testutils/capitest" + awsclusterconfig "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/aws/clusterconfig" + dockerclusterconfig "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/docker/clusterconfig" + nutanixclusterconfig "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/nutanix/clusterconfig" +) + +var testDefs = []capitest.VariableTestDef{ + { + Name: "without a credentials secret", + Vals: v1alpha1.GenericClusterConfigSpec{ + ImageRegistries: []v1alpha1.ImageRegistry{ + { + URL: "http://a.b.c.example.com", + }, + }, + }, + }, { + Name: "with a credentials secret", + Vals: v1alpha1.GenericClusterConfigSpec{ + ImageRegistries: []v1alpha1.ImageRegistry{ + { + URL: "https://a.b.c.example.com/a/b/c", + Credentials: &v1alpha1.RegistryCredentials{ + SecretRef: &v1alpha1.LocalObjectReference{ + Name: "a.b.c.example.com-creds", + }, + }, + }, + }, + }, + }, { + Name: "support for multiple image registries", + Vals: v1alpha1.GenericClusterConfigSpec{ + ImageRegistries: []v1alpha1.ImageRegistry{ + { + URL: "http://first-image-registry.example.com", + }, + { + URL: "http://second-image-registry.example.com", + }, + }, + }, + }, { + Name: "invalid registry URL", + Vals: v1alpha1.GenericClusterConfigSpec{ + ImageRegistries: []v1alpha1.ImageRegistry{ + { + URL: "unsupportedformat://a.b.c.example.com", + }, + }, + }, + ExpectError: true, + }, { + Name: "registry URL without format", + Vals: v1alpha1.GenericClusterConfigSpec{ + ImageRegistries: []v1alpha1.ImageRegistry{ + { + URL: "a.b.c.example.com/a/b/c", + }, + }, + }, + ExpectError: true, + }, +} + +func TestVariableValidation_AWS(t *testing.T) { + capitest.ValidateDiscoverVariables( + t, + v1alpha1.ClusterConfigVariableName, + ptr.To(v1alpha1.AWSClusterConfig{}.VariableSchema()), + true, + awsclusterconfig.NewVariable, + testDefs..., + ) +} + +func TestVariableValidation_Docker(t *testing.T) { + capitest.ValidateDiscoverVariables( + t, + v1alpha1.ClusterConfigVariableName, + ptr.To(v1alpha1.DockerClusterConfig{}.VariableSchema()), + true, + dockerclusterconfig.NewVariable, + testDefs..., + ) +} + +func TestVariableValidation_Nutanix(t *testing.T) { + capitest.ValidateDiscoverVariables( + t, + v1alpha1.ClusterConfigVariableName, + ptr.To(v1alpha1.NutanixClusterConfig{}.VariableSchema()), + true, + nutanixclusterconfig.NewVariable, + testDefs..., + ) +} diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files.go b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files.go index ee15ca6b7..8d6f8b865 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files.go @@ -9,8 +9,10 @@ import ( "fmt" "net/url" "path" + "sort" "text/template" + "github.com/samber/lo" corev1 "k8s.io/api/core/v1" credentialproviderv1 "k8s.io/kubelet/pkg/apis/credentialprovider/v1" cabpkv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" @@ -165,22 +167,30 @@ func templateKubeletCredentialProviderConfig( func templateDynamicCredentialProviderConfig( configs []providerConfig, ) (*cabpkv1.File, error) { - type templateInput struct { - RegistryHost string + type providerConfig struct { + RegistryHosts []string ProviderBinary string ProviderArgs []string ProviderAPIVersion string - Mirror bool + } + type templateInput struct { + Mirror string + ProviderConfigs []*providerConfig } - inputs := make([]templateInput, 0, len(configs)) + binaryToProviderConfigMap := map[string]*providerConfig{} + mirror := "" for _, config := range configs { registryHostWithPath, err := config.registryHostWithPath() if err != nil { return nil, err } + if config.Mirror { + mirror = registryHostWithPath + } + providerBinary, providerArgs, providerAPIVersion, err := dynamicCredentialProvider( registryHostWithPath, ) @@ -188,18 +198,34 @@ func templateDynamicCredentialProviderConfig( return nil, err } - inputs = append(inputs, templateInput{ - RegistryHost: registryHostWithPath, - ProviderBinary: providerBinary, - ProviderArgs: providerArgs, - ProviderAPIVersion: providerAPIVersion, - Mirror: config.Mirror, - }) + input, ok := binaryToProviderConfigMap[providerBinary] + if !ok { + input = &providerConfig{ + ProviderBinary: providerBinary, + ProviderArgs: providerArgs, + ProviderAPIVersion: providerAPIVersion, + } + binaryToProviderConfigMap[providerBinary] = input + } + + input.RegistryHosts = append(input.RegistryHosts, registryHostWithPath) + } + + // Make sure the output is deterministic to avoid unnecessary rollouts. + providerConfigs := lo.Values(binaryToProviderConfigMap) + for _, cfg := range providerConfigs { + sort.Strings(cfg.RegistryHosts) } + sort.SliceStable(providerConfigs, func(i, j int) bool { + return providerConfigs[i].ProviderBinary < providerConfigs[j].ProviderBinary + }) return fileFromTemplate( dynamicCredentialProviderConfigPatchTemplate, - inputs, + templateInput{ + Mirror: mirror, + ProviderConfigs: providerConfigs, + }, kubeletDynamicCredentialProviderConfigOnRemote, ) } diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files_test.go b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files_test.go index 75647e5a0..715f14f50 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files_test.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files_test.go @@ -118,6 +118,45 @@ providers: - "*.*.*.*.*.*" defaultCacheDuration: "0s" apiVersion: credentialprovider.kubelet.k8s.io/v1 +`, + }, + }, + { + name: "multiple image registries with static config", + credentials: []providerConfig{{ + URL: "https://myregistry.com:5000/myproject", + Username: "myuser", + Password: "mypassword", + }, { + URL: "https://myotherregistry.com:5000/myproject", + Username: "otheruser", + Password: "otherpassword", + }}, + want: &cabpkv1.File{ + Path: "/etc/kubernetes/image-credential-provider-config.yaml", + Owner: "", + Permissions: "0600", + Encoding: "", + Append: false, + Content: `apiVersion: kubelet.config.k8s.io/v1 +kind: CredentialProviderConfig +providers: +- name: dynamic-credential-provider + args: + - get-credentials + - -c + - /etc/kubernetes/dynamic-credential-provider-config.yaml + matchImages: + - "myregistry.com:5000/myproject" + - "myotherregistry.com:5000/myproject" + - "*" + - "*.*" + - "*.*.*" + - "*.*.*.*" + - "*.*.*.*.*" + - "*.*.*.*.*.*" + defaultCacheDuration: "0s" + apiVersion: credentialprovider.kubelet.k8s.io/v1 `, }, }, @@ -261,21 +300,6 @@ credentialProviders: apiVersion: kubelet.config.k8s.io/v1 kind: CredentialProviderConfig providers: - - name: static-credential-provider - args: - - /etc/kubernetes/static-image-credentials.json - matchImages: - - "registry-1.docker.io" - - "docker.io" - defaultCacheDuration: "0s" - apiVersion: credentialprovider.kubelet.k8s.io/v1 - - name: static-credential-provider - args: - - /etc/kubernetes/static-image-credentials.json - matchImages: - - "myregistry.com" - defaultCacheDuration: "0s" - apiVersion: credentialprovider.kubelet.k8s.io/v1 - name: ecr-credential-provider args: - get-credentials @@ -288,6 +312,9 @@ credentialProviders: - /etc/kubernetes/static-image-credentials.json matchImages: - "anotherregistry.com" + - "myregistry.com" + - "registry-1.docker.io" + - "docker.io" defaultCacheDuration: "0s" apiVersion: credentialprovider.kubelet.k8s.io/v1 `, @@ -325,12 +352,6 @@ credentialProviders: - get-credentials matchImages: - "123456789.dkr.ecr.us-east-1.amazonaws.com" - defaultCacheDuration: "0s" - apiVersion: credentialprovider.kubelet.k8s.io/v1 - - name: ecr-credential-provider - args: - - get-credentials - matchImages: - "98765432.dkr.ecr.us-east-1.amazonaws.com" defaultCacheDuration: "0s" apiVersion: credentialprovider.kubelet.k8s.io/v1 @@ -368,18 +389,12 @@ credentialProviders: apiVersion: kubelet.config.k8s.io/v1 kind: CredentialProviderConfig providers: - - name: static-credential-provider - args: - - /etc/kubernetes/static-image-credentials.json - matchImages: - - "myregistry.com" - defaultCacheDuration: "0s" - apiVersion: credentialprovider.kubelet.k8s.io/v1 - name: static-credential-provider args: - /etc/kubernetes/static-image-credentials.json matchImages: - "mymirror.com" + - "myregistry.com" defaultCacheDuration: "0s" apiVersion: credentialprovider.kubelet.k8s.io/v1 `, diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/templates/dynamic-credential-provider-config.yaml.gotmpl b/pkg/handlers/generic/mutation/imageregistries/credentials/templates/dynamic-credential-provider-config.yaml.gotmpl index 6bbf3fd95..4e8057643 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/templates/dynamic-credential-provider-config.yaml.gotmpl +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/templates/dynamic-credential-provider-config.yaml.gotmpl @@ -1,19 +1,16 @@ apiVersion: credentialprovider.d2iq.com/v1alpha1 kind: DynamicCredentialProviderConfig -{{- range .}} -{{- if .Mirror }} +{{- with .Mirror }} mirror: - endpoint: {{ .RegistryHost }} + endpoint: {{ . }} credentialsStrategy: MirrorCredentialsFirst -{{- break }} -{{- end }} {{- end }} credentialProviderPluginBinDir: /etc/kubernetes/image-credential-provider/ credentialProviders: apiVersion: kubelet.config.k8s.io/v1 kind: CredentialProviderConfig providers: - {{- range . }} + {{- range .ProviderConfigs }} - name: {{ .ProviderBinary }} {{- with .ProviderArgs }} args: @@ -22,7 +19,7 @@ credentialProviders: {{- end }} {{- end }} matchImages: - {{- with .RegistryHost }} + {{- range .RegistryHosts }} - {{ printf "%q" . }} {{- if eq . "registry-1.docker.io" }} - "docker.io"