fix: Correctly configure dynamic credential provider

It is only allowed for a single configuration per provider binary so match up
the image hosts per binary in the config.

Author: jimmidyson

make/dev.mk

@@ -32,7 +32,7 @@ dev.update-webhook-image-on-kind:
 	kind load docker-image --name $(KIND_CLUSTER_NAME) \
 	  ko.local/cluster-api-runtime-extensions-nutanix:$(SNAPSHOT_VERSION)
 	kubectl set image deployment \
-	  cluster-api-runtime-extensions-nutanix webhook=ko.local/cluster-api-runtime-extensions-nutanix:$(SNAPSHOT_VERSION)
+	  cluster-api-runtime-extensions-nutanix manager=ko.local/cluster-api-runtime-extensions-nutanix:$(SNAPSHOT_VERSION)
 	kubectl rollout restart deployment cluster-api-runtime-extensions-nutanix
 	kubectl rollout status deployment cluster-api-runtime-extensions-nutanix
 

pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files.go

@@ -9,8 +9,10 @@ import (
 	"fmt"
 	"net/url"
 	"path"
+	"sort"
 	"text/template"
 
+	"github.com/samber/lo"
 	corev1 "k8s.io/api/core/v1"
 	credentialproviderv1 "k8s.io/kubelet/pkg/apis/credentialprovider/v1"
 	cabpkv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
@@ -165,41 +167,65 @@ func templateKubeletCredentialProviderConfig(
 func templateDynamicCredentialProviderConfig(
 	configs []providerConfig,
 ) (*cabpkv1.File, error) {
-	type templateInput struct {
-		RegistryHost       string
+	type providerConfig struct {
+		RegistryHosts      []string
 		ProviderBinary     string
 		ProviderArgs       []string
 		ProviderAPIVersion string
-		Mirror             bool
+	}
+	type templateInput struct {
+		Mirror          string
+		ProviderConfigs []*providerConfig
 	}
 
-	inputs := make([]templateInput, 0, len(configs))
+	binaryToProviderConfigMap := map[string]*providerConfig{}
 
+	mirror := ""
 	for _, config := range configs {
 		registryHostWithPath, err := config.registryHostWithPath()
 		if err != nil {
 			return nil, err
 		}
 
+		if config.Mirror {
+			mirror = registryHostWithPath
+		}
+
 		providerBinary, providerArgs, providerAPIVersion, err := dynamicCredentialProvider(
 			registryHostWithPath,
 		)
 		if err != nil {
 			return nil, err
 		}
 
-		inputs = append(inputs, templateInput{
-			RegistryHost:       registryHostWithPath,
-			ProviderBinary:     providerBinary,
-			ProviderArgs:       providerArgs,
-			ProviderAPIVersion: providerAPIVersion,
-			Mirror:             config.Mirror,
-		})
+		input, ok := binaryToProviderConfigMap[providerBinary]
+		if !ok {
+			input = &providerConfig{
+				ProviderBinary:     providerBinary,
+				ProviderArgs:       providerArgs,
+				ProviderAPIVersion: providerAPIVersion,
+			}
+			binaryToProviderConfigMap[providerBinary] = input
+		}
+
+		input.RegistryHosts = append(input.RegistryHosts, registryHostWithPath)
+	}
+
+	// Make sure the output is deterministic to avoid unnecessary rollouts.
+	providerConfigs := lo.Values(binaryToProviderConfigMap)
+	for _, cfg := range providerConfigs {
+		sort.Strings(cfg.RegistryHosts)
 	}
+	sort.SliceStable(providerConfigs, func(i, j int) bool {
+		return providerConfigs[i].ProviderBinary < providerConfigs[j].ProviderBinary
+	})
 
 	return fileFromTemplate(
 		dynamicCredentialProviderConfigPatchTemplate,
-		inputs,
+		templateInput{
+			Mirror:          mirror,
+			ProviderConfigs: providerConfigs,
+		},
 		kubeletDynamicCredentialProviderConfigOnRemote,
 	)
 }

pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files_test.go

@@ -118,6 +118,45 @@ providers:
   - "*.*.*.*.*.*"
   defaultCacheDuration: "0s"
   apiVersion: credentialprovider.kubelet.k8s.io/v1
+`,
+			},
+		},
+		{
+			name: "multiple image registries with static config",
+			credentials: []providerConfig{{
+				URL:      "https://myregistry.com:5000/myproject",
+				Username: "myuser",
+				Password: "mypassword",
+			}, {
+				URL:      "https://myotherregistry.com:5000/myproject",
+				Username: "otheruser",
+				Password: "otherpassword",
+			}},
+			want: &cabpkv1.File{
+				Path:        "/etc/kubernetes/image-credential-provider-config.yaml",
+				Owner:       "",
+				Permissions: "0600",
+				Encoding:    "",
+				Append:      false,
+				Content: `apiVersion: kubelet.config.k8s.io/v1
+kind: CredentialProviderConfig
+providers:
+- name: dynamic-credential-provider
+  args:
+  - get-credentials
+  - -c
+  - /etc/kubernetes/dynamic-credential-provider-config.yaml
+  matchImages:
+  - "myregistry.com:5000/myproject"
+  - "myotherregistry.com:5000/myproject"
+  - "*"
+  - "*.*"
+  - "*.*.*"
+  - "*.*.*.*"
+  - "*.*.*.*.*"
+  - "*.*.*.*.*.*"
+  defaultCacheDuration: "0s"
+  apiVersion: credentialprovider.kubelet.k8s.io/v1
 `,
 			},
 		},
@@ -261,21 +300,6 @@ credentialProviders:
   apiVersion: kubelet.config.k8s.io/v1
   kind: CredentialProviderConfig
   providers:
-  - name: static-credential-provider
-    args:
-    - /etc/kubernetes/static-image-credentials.json
-    matchImages:
-    - "registry-1.docker.io"
-    - "docker.io"
-    defaultCacheDuration: "0s"
-    apiVersion: credentialprovider.kubelet.k8s.io/v1
-  - name: static-credential-provider
-    args:
-    - /etc/kubernetes/static-image-credentials.json
-    matchImages:
-    - "myregistry.com"
-    defaultCacheDuration: "0s"
-    apiVersion: credentialprovider.kubelet.k8s.io/v1
   - name: ecr-credential-provider
     args:
     - get-credentials
@@ -288,6 +312,9 @@ credentialProviders:
     - /etc/kubernetes/static-image-credentials.json
     matchImages:
     - "anotherregistry.com"
+    - "myregistry.com"
+    - "registry-1.docker.io"
+    - "docker.io"
     defaultCacheDuration: "0s"
     apiVersion: credentialprovider.kubelet.k8s.io/v1
 `,
@@ -325,12 +352,6 @@ credentialProviders:
     - get-credentials
     matchImages:
     - "123456789.dkr.ecr.us-east-1.amazonaws.com"
-    defaultCacheDuration: "0s"
-    apiVersion: credentialprovider.kubelet.k8s.io/v1
-  - name: ecr-credential-provider
-    args:
-    - get-credentials
-    matchImages:
     - "98765432.dkr.ecr.us-east-1.amazonaws.com"
     defaultCacheDuration: "0s"
     apiVersion: credentialprovider.kubelet.k8s.io/v1
@@ -368,18 +389,12 @@ credentialProviders:
   apiVersion: kubelet.config.k8s.io/v1
   kind: CredentialProviderConfig
   providers:
-  - name: static-credential-provider
-    args:
-    - /etc/kubernetes/static-image-credentials.json
-    matchImages:
-    - "myregistry.com"
-    defaultCacheDuration: "0s"
-    apiVersion: credentialprovider.kubelet.k8s.io/v1
   - name: static-credential-provider
     args:
     - /etc/kubernetes/static-image-credentials.json
     matchImages:
     - "mymirror.com"
+    - "myregistry.com"
     defaultCacheDuration: "0s"
     apiVersion: credentialprovider.kubelet.k8s.io/v1
 `,

pkg/handlers/generic/mutation/imageregistries/credentials/templates/dynamic-credential-provider-config.yaml.gotmpl

@@ -1,19 +1,16 @@
 apiVersion: credentialprovider.d2iq.com/v1alpha1
 kind: DynamicCredentialProviderConfig
-{{- range .}}
-{{- if .Mirror }}
+{{- with .Mirror }}
 mirror:
-  endpoint: {{ .RegistryHost }}
+  endpoint: {{ . }}
   credentialsStrategy: MirrorCredentialsFirst
-{{- break }}
-{{- end }}
 {{- end }}
 credentialProviderPluginBinDir: /etc/kubernetes/image-credential-provider/
 credentialProviders:
   apiVersion: kubelet.config.k8s.io/v1
   kind: CredentialProviderConfig
   providers:
-  {{- range . }}
+  {{- range .ProviderConfigs }}
   - name: {{ .ProviderBinary }}
     {{- with .ProviderArgs }}
     args:
@@ -22,7 +19,7 @@ credentialProviders:
     {{- end }}
     {{- end }}
     matchImages:
-    {{- with .RegistryHost }}
+    {{- range .RegistryHosts }}
     - {{ printf "%q" . }}
     {{- if eq . "registry-1.docker.io" }}
     - "docker.io"