fix: Correctly configgure dynamic credential provider

Only allowed a single configuration per provider binary so match up
the image hosts per binary in the config.

Tested and works well, including in conjunction with the CA config
in the previous PR.

Author: jimmidyson

make/dev.mk

@@ -32,7 +32,7 @@ dev.update-webhook-image-on-kind:
 	kind load docker-image --name $(KIND_CLUSTER_NAME) \
 	  ko.local/cluster-api-runtime-extensions-nutanix:$(SNAPSHOT_VERSION)
 	kubectl set image deployment \
-	  cluster-api-runtime-extensions-nutanix webhook=ko.local/cluster-api-runtime-extensions-nutanix:$(SNAPSHOT_VERSION)
+	  cluster-api-runtime-extensions-nutanix manager=ko.local/cluster-api-runtime-extensions-nutanix:$(SNAPSHOT_VERSION)
 	kubectl rollout restart deployment cluster-api-runtime-extensions-nutanix
 	kubectl rollout status deployment cluster-api-runtime-extensions-nutanix
 

pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files.go

@@ -9,8 +9,10 @@ import (
 	"fmt"
 	"net/url"
 	"path"
+	"sort"
 	"text/template"
 
+	"github.com/samber/lo"
 	corev1 "k8s.io/api/core/v1"
 	credentialproviderv1 "k8s.io/kubelet/pkg/apis/credentialprovider/v1"
 	cabpkv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
@@ -165,41 +167,65 @@ func templateKubeletCredentialProviderConfig(
 func templateDynamicCredentialProviderConfig(
 	configs []providerConfig,
 ) (*cabpkv1.File, error) {
-	type templateInput struct {
-		RegistryHost       string
+	type providerConfig struct {
+		RegistryHosts      []string
 		ProviderBinary     string
 		ProviderArgs       []string
 		ProviderAPIVersion string
-		Mirror             bool
+	}
+	type templateInput struct {
+		Mirror          string
+		ProviderConfigs []*providerConfig
 	}
 
-	inputs := make([]templateInput, 0, len(configs))
+	binaryToProviderConfigMap := map[string]*providerConfig{}
 
+	mirror := ""
 	for _, config := range configs {
 		registryHostWithPath, err := config.registryHostWithPath()
 		if err != nil {
 			return nil, err
 		}
 
+		if config.Mirror {
+			mirror = registryHostWithPath
+		}
+
 		providerBinary, providerArgs, providerAPIVersion, err := dynamicCredentialProvider(
 			registryHostWithPath,
 		)
 		if err != nil {
 			return nil, err
 		}
 
-		inputs = append(inputs, templateInput{
-			RegistryHost:       registryHostWithPath,
-			ProviderBinary:     providerBinary,
-			ProviderArgs:       providerArgs,
-			ProviderAPIVersion: providerAPIVersion,
-			Mirror:             config.Mirror,
-		})
+		input, ok := binaryToProviderConfigMap[providerBinary]
+		if !ok {
+			input = &providerConfig{
+				ProviderBinary:     providerBinary,
+				ProviderArgs:       providerArgs,
+				ProviderAPIVersion: providerAPIVersion,
+			}
+			binaryToProviderConfigMap[providerBinary] = input
+		}
+
+		input.RegistryHosts = append(input.RegistryHosts, registryHostWithPath)
+	}
+
+	// Make sure the output is deterministic to avoid unnecessary rollouts.
+	providerConfigs := lo.Values(binaryToProviderConfigMap)
+	for _, cfg := range providerConfigs {
+		sort.Strings(cfg.RegistryHosts)
 	}
+	sort.SliceStable(providerConfigs, func(i, j int) bool {
+		return providerConfigs[i].ProviderBinary < providerConfigs[j].ProviderBinary
+	})
 
 	return fileFromTemplate(
 		dynamicCredentialProviderConfigPatchTemplate,
-		inputs,
+		templateInput{
+			Mirror:          mirror,
+			ProviderConfigs: providerConfigs,
+		},
 		kubeletDynamicCredentialProviderConfigOnRemote,
 	)
 }

pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files_test.go

@@ -118,6 +118,45 @@ providers:
   - "*.*.*.*.*.*"
   defaultCacheDuration: "0s"
   apiVersion: credentialprovider.kubelet.k8s.io/v1
+`,
+			},
+		},
+		{
+			name: "multiple image registries with static config",
+			credentials: []providerConfig{{
+				URL:      "https://myregistry.com:5000/myproject",
+				Username: "myuser",
+				Password: "mypassword",
+			}, {
+				URL:      "https://myotherregistry.com:5000/myproject",
+				Username: "otheruser",
+				Password: "otherpassword",
+			}},
+			want: &cabpkv1.File{
+				Path:        "/etc/kubernetes/image-credential-provider-config.yaml",
+				Owner:       "",
+				Permissions: "0600",
+				Encoding:    "",
+				Append:      false,
+				Content: `apiVersion: kubelet.config.k8s.io/v1
+kind: CredentialProviderConfig
+providers:
+- name: dynamic-credential-provider
+  args:
+  - get-credentials
+  - -c
+  - /etc/kubernetes/dynamic-credential-provider-config.yaml
+  matchImages:
+  - "myregistry.com:5000/myproject"
+  - "myotherregistry.com:5000/myproject"
+  - "*"
+  - "*.*"
+  - "*.*.*"
+  - "*.*.*.*"
+  - "*.*.*.*.*"
+  - "*.*.*.*.*.*"
+  defaultCacheDuration: "0s"
+  apiVersion: credentialprovider.kubelet.k8s.io/v1
 `,
 			},
 		},
@@ -261,21 +300,6 @@ credentialProviders:
   apiVersion: kubelet.config.k8s.io/v1
   kind: CredentialProviderConfig
   providers:
-  - name: static-credential-provider
-    args:
-    - /etc/kubernetes/static-image-credentials.json
-    matchImages:
-    - "registry-1.docker.io"
-    - "docker.io"
-    defaultCacheDuration: "0s"
-    apiVersion: credentialprovider.kubelet.k8s.io/v1
-  - name: static-credential-provider
-    args:
-    - /etc/kubernetes/static-image-credentials.json
-    matchImages:
-    - "myregistry.com"
-    defaultCacheDuration: "0s"
-    apiVersion: credentialprovider.kubelet.k8s.io/v1
   - name: ecr-credential-provider
     args:
     - get-credentials
@@ -288,6 +312,9 @@ credentialProviders:
     - /etc/kubernetes/static-image-credentials.json
     matchImages:
     - "anotherregistry.com"
+    - "myregistry.com"
+    - "registry-1.docker.io"
+    - "docker.io"
     defaultCacheDuration: "0s"
     apiVersion: credentialprovider.kubelet.k8s.io/v1
 `,
@@ -325,12 +352,6 @@ credentialProviders:
     - get-credentials
     matchImages:
     - "123456789.dkr.ecr.us-east-1.amazonaws.com"
-    defaultCacheDuration: "0s"
-    apiVersion: credentialprovider.kubelet.k8s.io/v1
-  - name: ecr-credential-provider
-    args:
-    - get-credentials
-    matchImages:
     - "98765432.dkr.ecr.us-east-1.amazonaws.com"
     defaultCacheDuration: "0s"
     apiVersion: credentialprovider.kubelet.k8s.io/v1
@@ -368,18 +389,12 @@ credentialProviders:
   apiVersion: kubelet.config.k8s.io/v1
   kind: CredentialProviderConfig
   providers:
-  - name: static-credential-provider
-    args:
-    - /etc/kubernetes/static-image-credentials.json
-    matchImages:
-    - "myregistry.com"
-    defaultCacheDuration: "0s"
-    apiVersion: credentialprovider.kubelet.k8s.io/v1
   - name: static-credential-provider
     args:
     - /etc/kubernetes/static-image-credentials.json
     matchImages:
     - "mymirror.com"
+    - "myregistry.com"
     defaultCacheDuration: "0s"
     apiVersion: credentialprovider.kubelet.k8s.io/v1
 `,

pkg/handlers/generic/mutation/imageregistries/credentials/templates/dynamic-credential-provider-config.yaml.gotmpl

@@ -1,19 +1,16 @@
 apiVersion: credentialprovider.d2iq.com/v1alpha1
 kind: DynamicCredentialProviderConfig
-{{- range .}}
-{{- if .Mirror }}
+{{- with .Mirror }}
 mirror:
-  endpoint: {{ .RegistryHost }}
+  endpoint: {{ . }}
   credentialsStrategy: MirrorCredentialsFirst
-{{- break }}
-{{- end }}
 {{- end }}
 credentialProviderPluginBinDir: /etc/kubernetes/image-credential-provider/
 credentialProviders:
   apiVersion: kubelet.config.k8s.io/v1
   kind: CredentialProviderConfig
   providers:
-  {{- range . }}
+  {{- range .ProviderConfigs }}
   - name: {{ .ProviderBinary }}
     {{- with .ProviderArgs }}
     args:
@@ -22,7 +19,7 @@ credentialProviders:
     {{- end }}
     {{- end }}
     matchImages:
-    {{- with .RegistryHost }}
+    {{- range .RegistryHosts }}
     - {{ printf "%q" . }}
     {{- if eq . "registry-1.docker.io" }}
     - "docker.io"