feat: Enable configurable parallel image pulls (#1306)

jimmidyson · web-flow · commit 64bc043a88d9 · 2025-09-22T19:31:40.000Z
**What problem does this PR solve?**:

Default behaviour is retained, that is serialized image pulls actually
configured by disabling forced serialized image pulls and setting max
parallel image pulls to 1.

A value of zero means unlimited parallel image pulls.


**Which issue(s) this PR fixes**:
Fixes #

**How Has This Been Tested?**:
&lt;!--
Please describe the tests that you ran to verify your changes.
Provide output from the tests and any manual steps needed to replicate
the tests.
--&gt;

**Special notes for your reviewer**:
&lt;!--
Use this to provide any additional information to the reviewers.
This may include:
- Best way to review the PR.
- Where the author wants the most review attention on.
- etc.
--&gt;
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -81,7 +81,7 @@ repos:
   - id: check-yaml
     args: ["-m", "--unsafe"]
     stages: [pre-commit]
-    exclude: ^charts/.+/(templates|addons)/.+\.ya?ml$
+    exclude: (embedded/kubeletconfigpatch.yaml|^charts/.+/(templates|addons)/.+\.ya?ml)$
   - id: mixed-line-ending
     args: ["-f", "lf"]
     exclude: \.bat$
diff --git a/api/v1alpha1/clusterconfig_types.go b/api/v1alpha1/clusterconfig_types.go
@@ -246,6 +246,13 @@ type KubeadmClusterConfigSpec struct {
 	// KubeProxy defines the configuration for kube-proxy.
 	// +kubebuilder:validation:Optional
 	KubeProxy *KubeProxy `json:"kubeProxy,omitempty"`
+
+	// MaxParallelImagePullsPerNode defines the maximum number of parallel image pulls performed by each kubelet.
+	// If not set, the default value of 1 will be used.
+	// If set to 0, the maximum number of parallel image pulls will be unlimited.
+	// +kubebuilder:validation:Optional
+	// +kubebuilder:validation:Minimum=0
+	MaxParallelImagePullsPerNode *int32 `json:"maxParallelImagePullsPerNode,omitempty"`
 }
 
 // +kubebuilder:object:root=true
diff --git a/api/v1alpha1/crds/caren.nutanix.com_awsclusterconfigs.yaml b/api/v1alpha1/crds/caren.nutanix.com_awsclusterconfigs.yaml
@@ -636,6 +636,14 @@ spec:
                   minLength: 1
                   pattern: ^((?:[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*|\[(?:[a-fA-F0-9:]+)\])(:[0-9]+)?/)?[a-z0-9]+((?:[._]|__|[-]+)[a-z0-9]+)*(/[a-z0-9]+((?:[._]|__|[-]+)[a-z0-9]+)*)*$
                   type: string
+                maxParallelImagePullsPerNode:
+                  description: |-
+                    MaxParallelImagePullsPerNode defines the maximum number of parallel image pulls performed by each kubelet.
+                    If not set, the default value of 1 will be used.
+                    If set to 0, the maximum number of parallel image pulls will be unlimited.
+                  format: int32
+                  minimum: 0
+                  type: integer
                 ntp:
                   description: NTP defines the NTP configuration for the cluster.
                   properties:
diff --git a/api/v1alpha1/crds/caren.nutanix.com_dockerclusterconfigs.yaml b/api/v1alpha1/crds/caren.nutanix.com_dockerclusterconfigs.yaml
@@ -545,6 +545,14 @@ spec:
                   minLength: 1
                   pattern: ^((?:[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*|\[(?:[a-fA-F0-9:]+)\])(:[0-9]+)?/)?[a-z0-9]+((?:[._]|__|[-]+)[a-z0-9]+)*(/[a-z0-9]+((?:[._]|__|[-]+)[a-z0-9]+)*)*$
                   type: string
+                maxParallelImagePullsPerNode:
+                  description: |-
+                    MaxParallelImagePullsPerNode defines the maximum number of parallel image pulls performed by each kubelet.
+                    If not set, the default value of 1 will be used.
+                    If set to 0, the maximum number of parallel image pulls will be unlimited.
+                  format: int32
+                  minimum: 0
+                  type: integer
                 ntp:
                   description: NTP defines the NTP configuration for the cluster.
                   properties:
diff --git a/api/v1alpha1/crds/caren.nutanix.com_kubeadmclusterconfigs.yaml b/api/v1alpha1/crds/caren.nutanix.com_kubeadmclusterconfigs.yaml
@@ -134,6 +134,14 @@ spec:
                 minLength: 1
                 pattern: ^((?:[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*|\[(?:[a-fA-F0-9:]+)\])(:[0-9]+)?/)?[a-z0-9]+((?:[._]|__|[-]+)[a-z0-9]+)*(/[a-z0-9]+((?:[._]|__|[-]+)[a-z0-9]+)*)*$
                 type: string
+              maxParallelImagePullsPerNode:
+                description: |-
+                  MaxParallelImagePullsPerNode defines the maximum number of parallel image pulls performed by each kubelet.
+                  If not set, the default value of 1 will be used.
+                  If set to 0, the maximum number of parallel image pulls will be unlimited.
+                format: int32
+                minimum: 0
+                type: integer
             type: object
         type: object
     served: true
diff --git a/api/v1alpha1/crds/caren.nutanix.com_nutanixclusterconfigs.yaml b/api/v1alpha1/crds/caren.nutanix.com_nutanixclusterconfigs.yaml
@@ -733,6 +733,14 @@ spec:
                   minLength: 1
                   pattern: ^((?:[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*|\[(?:[a-fA-F0-9:]+)\])(:[0-9]+)?/)?[a-z0-9]+((?:[._]|__|[-]+)[a-z0-9]+)*(/[a-z0-9]+((?:[._]|__|[-]+)[a-z0-9]+)*)*$
                   type: string
+                maxParallelImagePullsPerNode:
+                  description: |-
+                    MaxParallelImagePullsPerNode defines the maximum number of parallel image pulls performed by each kubelet.
+                    If not set, the default value of 1 will be used.
+                    If set to 0, the maximum number of parallel image pulls will be unlimited.
+                  format: int32
+                  minimum: 0
+                  type: integer
                 ntp:
                   description: NTP defines the NTP configuration for the cluster.
                   properties:
diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
diff --git a/docs/content/customization/generic/parallel-image-pulls.md b/docs/content/customization/generic/parallel-image-pulls.md
@@ -0,0 +1,66 @@
++++
+title = "Parallel Image Pulls"
++++
+
+This customization will be available when the
+[provider-specific cluster configuration patch]({{< ref "..">}}) is included in the `ClusterClass`.
+
+The parallel image pull configuration can then be manipulated via the cluster variables.
+If the `maxParallelImagePullsPerNode` property is not specified, then the default value of `1` will be used
+which is equivalent to serialized image pulls.
+
+Setting this value to `0` results in unlimited parallel image pulls.
+
+### Example
+
+```yaml
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: Cluster
+metadata:
+  name: <NAME>
+spec:
+  topology:
+    variables:
+      - name: clusterConfig
+        value:
+          maxParallelImagePullsPerNodePerNode: 10
+```
+
+Applying this configuration will result in a `KubeletConfiguration` patch being added which will be
+applied by `kubeadm` on `init` and `join`:
+
+- `KubeadmControlPlaneTemplate`:
+
+  - ```yaml
+    spec:
+      template:
+        spec:
+          kubeadmConfigSpec:
+            files:
+              - path: "/etc/kubernetes/patches/kubeletconfigurationmaxparallelimagepulls+strategic.json"
+                owner: "root:root"
+                permissions: "0644"
+                content: |-
+                  ---
+                  apiVersion: kubelet.config.k8s.io/v1beta1
+                  kind: KubeletConfiguration
+                  serializeImagePulls: false
+                  maxParallelImagePulls: 10
+    ```
+
+- `KubeadmConfigTemplate`
+
+  - ```yaml
+    spec:
+      kubeadmConfigSpec:
+        files:
+          - path: "/etc/kubernetes/patches/kubeletconfigurationmaxparallelimagepulls+strategic.json"
+            owner: "root:root"
+            permissions: "0644"
+            content: |-
+              ---
+              apiVersion: kubelet.config.k8s.io/v1beta1
+              kind: KubeletConfiguration
+              serializeImagePulls: false
+              maxParallelImagePulls: 10
+    ```
diff --git a/pkg/handlers/generic/mutation/handlers.go b/pkg/handlers/generic/mutation/handlers.go
@@ -25,6 +25,7 @@ import (
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/mirrors"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/noderegistration"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/ntp"
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/parallelimagepulls"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/taints"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/users"
 )
@@ -48,6 +49,7 @@ func MetaMutators(mgr manager.Manager) []mutation.MetaMutator {
 		autorenewcerts.NewPatch(),
 		kubeproxymode.NewPatch(),
 		ntp.NewPatch(),
+		parallelimagepulls.NewPatch(),
 
 		// Some patches may have changed containerd configuration.
 		// We write the configuration changes to disk, and must run a command
diff --git a/pkg/handlers/generic/mutation/parallelimagepulls/embedded/kubeletconfigpatch.yaml b/pkg/handlers/generic/mutation/parallelimagepulls/embedded/kubeletconfigpatch.yaml
@@ -0,0 +1,7 @@
+---
+apiVersion: kubelet.config.k8s.io/v1beta1
+kind: KubeletConfiguration
+serializeImagePulls: false
+{{- if gt .MaxParallelImagePullsPerNode 0 }}
+maxParallelImagePulls: {{ .MaxParallelImagePullsPerNode }}
+{{- end }}
diff --git a/pkg/handlers/generic/mutation/parallelimagepulls/inject.go b/pkg/handlers/generic/mutation/parallelimagepulls/inject.go
@@ -0,0 +1,179 @@
+// Copyright 2025 Nutanix. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package parallelimagepulls
+
+import (
+	"bytes"
+	"context"
+	_ "embed"
+	"fmt"
+	"text/template"
+
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
+	controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1"
+	runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1"
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers/mutation"
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/patches"
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/patches/selectors"
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/variables"
+)
+
+const (
+	// VariableName is the external patch variable name.
+	VariableName = "maxParallelImagePullsPerNode"
+
+	kubeletConfigurationPatchFilePath = "/etc/kubernetes/patches/kubeletconfigurationmaxparallelimagepulls+strategic.json"
+)
+
+var (
+	//go:embed embedded/kubeletconfigpatch.yaml
+	kubeletConfigPatchYAML []byte
+
+	kubeletConfigPatchTemplate = template.Must(template.New("kubeletConfigPatch").Parse(string(kubeletConfigPatchYAML)))
+)
+
+type maxParallelImagePullsPerNode struct {
+	variableName      string
+	variableFieldPath []string
+}
+
+func NewPatch() *maxParallelImagePullsPerNode {
+	return newMaxParallelImagePullsPerNodePatch(
+		v1alpha1.ClusterConfigVariableName,
+		VariableName,
+	)
+}
+
+func newMaxParallelImagePullsPerNodePatch(
+	variableName string,
+	variableFieldPath ...string,
+) *maxParallelImagePullsPerNode {
+	return &maxParallelImagePullsPerNode{
+		variableName:      variableName,
+		variableFieldPath: variableFieldPath,
+	}
+}
+
+func (h *maxParallelImagePullsPerNode) Mutate(
+	ctx context.Context,
+	obj *unstructured.Unstructured,
+	vars map[string]apiextensionsv1.JSON,
+	holderRef runtimehooksv1.HolderReference,
+	_ client.ObjectKey,
+	clusterGetter mutation.ClusterGetter,
+) error {
+	log := ctrl.LoggerFrom(ctx).WithValues(
+		"holderRef", holderRef,
+	)
+
+	maxParallelImagePullsPerNode, err := variables.Get[int32](
+		vars,
+		h.variableName,
+		h.variableFieldPath...,
+	)
+	if err != nil {
+		if variables.IsNotFoundError(err) {
+			log.V(5).Info("max parallel image pulls is not set, skipping mutation")
+			return nil
+		}
+		return err
+	}
+
+	if maxParallelImagePullsPerNode == 1 {
+		log.V(5).Info("max parallel image pulls is set to 1, skipping mutation resulting in serialized image pulls")
+		return nil
+	}
+
+	log = log.WithValues(
+		"variableName",
+		h.variableName,
+		"variableFieldPath",
+		h.variableFieldPath,
+		"variableValue",
+		maxParallelImagePullsPerNode,
+	)
+
+	kubeletConfigPatch, err := templateMaxParallelImagePullsPerNodeConfigFile(maxParallelImagePullsPerNode)
+	if err != nil {
+		return err
+	}
+
+	if err := patches.MutateIfApplicable(
+		obj,
+		vars,
+		&holderRef,
+		selectors.ControlPlane(),
+		log,
+		func(obj *controlplanev1.KubeadmControlPlaneTemplate) error {
+			log.WithValues(
+				"patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(),
+				"patchedObjectName", client.ObjectKeyFromObject(obj),
+			).Info("adding max parallel image pulls patch to control plane kubeadm config spec")
+
+			obj.Spec.Template.Spec.KubeadmConfigSpec.Files = append(
+				obj.Spec.Template.Spec.KubeadmConfigSpec.Files,
+				*kubeletConfigPatch,
+			)
+
+			return nil
+		},
+	); err != nil {
+		return err
+	}
+
+	if err := patches.MutateIfApplicable(
+		obj,
+		vars,
+		&holderRef,
+		selectors.WorkersKubeadmConfigTemplateSelector(),
+		log,
+		func(obj *bootstrapv1.KubeadmConfigTemplate) error {
+			log.WithValues(
+				"patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(),
+				"patchedObjectName", client.ObjectKeyFromObject(obj),
+			).Info("adding max parallel image pulls patch to worker node kubeadm config template")
+
+			obj.Spec.Template.Spec.Files = append(
+				obj.Spec.Template.Spec.Files,
+				*kubeletConfigPatch,
+			)
+
+			return nil
+		},
+	); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// templateMaxParallelImagePullsPerNodeConfigFile adds the max parallel image pulls configuration patch file
+// to the KCPTemplate.
+func templateMaxParallelImagePullsPerNodeConfigFile(
+	maxParallelImagePullsPerNode int32,
+) (*bootstrapv1.File, error) {
+	templateInput := struct {
+		MaxParallelImagePullsPerNode int32
+	}{
+		MaxParallelImagePullsPerNode: maxParallelImagePullsPerNode,
+	}
+	var b bytes.Buffer
+	err := kubeletConfigPatchTemplate.Execute(&b, templateInput)
+	if err != nil {
+		return nil, fmt.Errorf("failed executing kubeletconfig patch template: %w", err)
+	}
+
+	return &bootstrapv1.File{
+		Path:        kubeletConfigurationPatchFilePath,
+		Owner:       "root:root",
+		Permissions: "0644",
+		Content:     b.String(),
+	}, nil
+}
diff --git a/pkg/handlers/generic/mutation/parallelimagepulls/inject_test.go b/pkg/handlers/generic/mutation/parallelimagepulls/inject_test.go
diff --git a/pkg/handlers/generic/mutation/parallelimagepulls/variables_test.go b/pkg/handlers/generic/mutation/parallelimagepulls/variables_test.go
