✨ Add scale from/to 0 support for CAPD (#12572)

* Add scale from 0 support for CAPD

* Add devcluster template reconciler

Author: Karthik-K-N

test/e2e/autoscaler.go

@@ -260,9 +260,7 @@ func AutoscalerSpec(ctx context.Context, inputGetter func() AutoscalerSpecInput)
 			By("Scaling the MachineDeployment scale up deployment to zero")
 			framework.ScaleScaleUpDeploymentAndWait(ctx, framework.ScaleScaleUpDeploymentAndWaitInput{
 				ClusterProxy: workloadClusterProxy,
-				// We need to sum up the expected number of MachineDeployment replicas and the current
-				// number of MachinePool replicas because otherwise the pods get scheduled on the MachinePool nodes.
-				Replicas: mpOriginalReplicas + 0,
+				Replicas:     0,
 			}, input.E2EConfig.GetIntervals(specName, "wait-autoscaler")...)
 
 			By("Checking the MachineDeployment finished scaling down to zero")

test/e2e/autoscaler_test.go

@@ -36,7 +36,8 @@ var _ = Describe("When using the autoscaler with Cluster API using ClusterClass
 			InfrastructureMachinePoolTemplateKind: "dockermachinepooltemplates",
 			InfrastructureMachinePoolKind:         "dockermachinepools",
 			Flavor:                                ptr.To("topology-autoscaler"),
-			AutoscalerVersion:                     "v1.33.0",
+			AutoscalerVersion:                     "v1.32.1",
+			ScaleToAndFromZero:                    true,
 		}
 	})
 })

test/e2e/config/docker.yaml

@@ -401,7 +401,7 @@ variables:
   CAPI_INSECURE_DIAGNOSTICS: "true"
 
 intervals:
-  default/wait-controllers: ["3m", "10s"]
+  default/wait-controllers: ["4m", "10s"]
   default/wait-cluster: ["5m", "10s"]
   default/wait-control-plane: ["10m", "10s"]
   default/wait-worker-nodes: ["5m", "10s"]

test/e2e/data/autoscaler/autoscaler-to-workload-workload.yaml

@@ -198,6 +198,8 @@ spec:
             # Note: The E2E test should only go up to 4 (assuming it starts with a min node group size of 2).
             # Using 6 for additional some buffer and to allow different starting min node group sizes.
             - --max-nodes-total=6
+            # Set the log verbosity
+            - --v=4
           env:
             # Per default autoscaler uses the preferred apiVersion to retrieve MachineDeployments.
             # If that apiVersion is v1beta2 the current autoscaler implementation is not able
@@ -208,6 +210,13 @@ spec:
             - name: kubeconfig-management-cluster
               mountPath: /management-cluster
               readOnly: true
+      # Run the autoscaler on control plane Machines to avoid disruptions when scaling to 0.
+      nodeSelector:
+        node-role.kubernetes.io/control-plane: ""
+      tolerations:
+        - key: node-role.kubernetes.io/control-plane
+          effect: NoSchedule
+          operator: Exists
       serviceAccountName: cluster-autoscaler
       terminationGracePeriodSeconds: 10
       volumes:

test/infrastructure/container/docker.go

@@ -64,6 +64,7 @@ func NewDockerClient() (Runtime, error) {
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to created docker runtime client")
 	}
+
 	return &dockerRuntime{
 		dockerClient: dockerClient,
 	}, nil
@@ -544,6 +545,11 @@ func (d *dockerRuntime) RunContainer(ctx context.Context, runConfig *RunContaine
 	return nil
 }
 
+// GetSystemInfo will return the docker system info.
+func (d *dockerRuntime) GetSystemInfo(ctx context.Context) (dockersystem.Info, error) {
+	return d.dockerClient.Info(ctx)
+}
+
 // needsDevMapper checks whether we need to mount /dev/mapper.
 // This is required when the docker storage driver is Btrfs or ZFS.
 // https://github.com/kubernetes-sigs/kind/pull/1464

test/infrastructure/container/fake.go

@@ -19,6 +19,8 @@ package container
 import (
 	"context"
 	"io"
+
+	dockersystem "github.com/docker/docker/api/types/system"
 )
 
 var runContainerCallLog []RunContainerArgs
@@ -170,3 +172,8 @@ func (f *FakeRuntime) RunContainerCalls() []RunContainerArgs {
 func (f *FakeRuntime) ResetRunContainerCallLogs() {
 	runContainerCallLog = []RunContainerArgs{}
 }
+
+// GetSystemInfo returns empty docker system info.
+func (f *FakeRuntime) GetSystemInfo(_ context.Context) (dockersystem.Info, error) {
+	return dockersystem.Info{}, nil
+}

test/infrastructure/container/interface.go

@@ -23,6 +23,7 @@ import (
 	"net"
 
 	dockercontainer "github.com/docker/docker/api/types/container"
+	dockersystem "github.com/docker/docker/api/types/system"
 	"github.com/pkg/errors"
 
 	clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2"
@@ -46,6 +47,7 @@ type Runtime interface {
 	ContainerDebugInfo(ctx context.Context, containerName string, w io.Writer) error
 	DeleteContainer(ctx context.Context, containerName string) error
 	KillContainer(ctx context.Context, containerName, signal string) error
+	GetSystemInfo(ctx context.Context) (dockersystem.Info, error)
 }
 
 // Mount contains mount details.

test/infrastructure/docker/api/v1alpha3/conversion.go

@@ -165,6 +165,7 @@ func (src *DockerMachineTemplate) ConvertTo(dstRaw conversion.Hub) error {
 
 	if ok {
 		RestoreDockerMachineTemplateSpec(&restored.Spec, &dst.Spec)
+		dst.Status = restored.Status
 	}
 
 	return nil
@@ -350,3 +351,7 @@ func Convert_v1alpha3_DockerClusterSpec_To_v1beta2_DockerClusterSpec(in *DockerC
 
 	return nil
 }
+
+func Convert_v1beta2_DockerMachineTemplate_To_v1alpha3_DockerMachineTemplate(in *infrav1.DockerMachineTemplate, out *DockerMachineTemplate, s apiconversion.Scope) error {
+	return autoConvert_v1beta2_DockerMachineTemplate_To_v1alpha3_DockerMachineTemplate(in, out, s)
+}

test/infrastructure/docker/api/v1alpha3/zz_generated.conversion.go

@@ -196,6 +196,7 @@ func (src *DockerMachineTemplate) ConvertTo(dstRaw conversion.Hub) error {
 
 	if ok {
 		RestoreDockerMachineTemplateSpec(&restored.Spec, &dst.Spec)
+		dst.Status = restored.Status
 	}
 
 	return nil