Merge pull request kubernetes-sigs#7271 from sbueringer/pr-make-CAPD-bootstrap-reentrant

🌱 CAPD: make Machine bootstrap reentrant

Author: k8s-ci-robot

test/infrastructure/docker/api/v1beta1/dockermachine_types.go

@@ -51,6 +51,7 @@ type DockerMachineSpec struct {
 
 	// Bootstrapped is true when the kubeadm bootstrapping has been run
 	// against this machine
+	// Deprecated: This field will be removed in the next apiVersion.
 	// +optional
 	Bootstrapped bool `json:"bootstrapped,omitempty"`
 }

test/infrastructure/docker/config/crd/bases/infrastructure.cluster.x-k8s.io_dockermachinepools.yaml

@@ -107,6 +107,7 @@ type DockerMachinePoolInstanceStatus struct {
 
 	// Bootstrapped is true when the kubeadm bootstrapping has been run
 	// against this machine
+	// Deprecated: This field will be removed in the next apiVersion.
 	// +optional
 	Bootstrapped bool `json:"bootstrapped,omitempty"`
 }

test/infrastructure/docker/config/crd/bases/infrastructure.cluster.x-k8s.io_dockermachines.yaml

@@ -278,28 +278,35 @@ func (np *NodePool) reconcileMachine(ctx context.Context, machine *docker.Machin
 
 	// if the machine isn't bootstrapped, only then run bootstrap scripts
 	if !machineStatus.Bootstrapped {
-		log.Info("Bootstrapping instance", "instance", machine.Name())
-		if err := externalMachine.PreloadLoadImages(ctx, np.dockerMachinePool.Spec.Template.PreLoadImages); err != nil {
-			return ctrl.Result{}, errors.Wrapf(err, "failed to pre-load images into the docker machine with instance name %s", machine.Name())
-		}
-
-		bootstrapData, format, err := getBootstrapData(ctx, np.client, np.machinePool)
-		if err != nil {
-			return ctrl.Result{}, errors.Wrapf(err, "failed to get bootstrap data for instance named %s", machine.Name())
-		}
-
-		timeoutctx, cancel := context.WithTimeout(ctx, 3*time.Minute)
+		timeoutCtx, cancel := context.WithTimeout(ctx, 3*time.Minute)
 		defer cancel()
-		// Run the bootstrap script. Simulates cloud-init/Ignition.
-		if err := externalMachine.ExecBootstrap(timeoutctx, bootstrapData, format); err != nil {
-			return ctrl.Result{}, errors.Wrapf(err, "failed to exec DockerMachinePool instance bootstrap for instance named %s", machine.Name())
-		}
+
 		// Check for bootstrap success
-		if err := externalMachine.CheckForBootstrapSuccess(timeoutctx); err != nil {
-			return ctrl.Result{}, errors.Wrap(err, "failed to check for existence of bootstrap success file at /run/cluster-api/bootstrap-success.complete")
-		}
+		// We have to check here to make this reentrant for cases where the bootstrap works
+		// but bootstrapped is never set on the object. We only try to bootstrap if the machine
+		// is not already bootstrapped.
+		if err := externalMachine.CheckForBootstrapSuccess(timeoutCtx, false); err != nil {
+			log.Info("Bootstrapping instance", "instance", machine.Name())
+			if err := externalMachine.PreloadLoadImages(timeoutCtx, np.dockerMachinePool.Spec.Template.PreLoadImages); err != nil {
+				return ctrl.Result{}, errors.Wrapf(err, "failed to pre-load images into the docker machine with instance name %s", machine.Name())
+			}
+
+			bootstrapData, format, err := getBootstrapData(timeoutCtx, np.client, np.machinePool)
+			if err != nil {
+				return ctrl.Result{}, errors.Wrapf(err, "failed to get bootstrap data for instance named %s", machine.Name())
+			}
 
+			// Run the bootstrap script. Simulates cloud-init/Ignition.
+			if err := externalMachine.ExecBootstrap(timeoutCtx, bootstrapData, format); err != nil {
+				return ctrl.Result{}, errors.Wrapf(err, "failed to exec DockerMachinePool instance bootstrap for instance named %s", machine.Name())
+			}
+			// Check for bootstrap success
+			if err := externalMachine.CheckForBootstrapSuccess(timeoutCtx, true); err != nil {
+				return ctrl.Result{}, errors.Wrap(err, "failed to check for existence of bootstrap success file at /run/cluster-api/bootstrap-success.complete")
+			}
+		}
 		machineStatus.Bootstrapped = true
+
 		// return to surface the machine has been bootstrapped.
 		return ctrl.Result{Requeue: true}, nil
 	}

test/infrastructure/docker/config/crd/bases/infrastructure.cluster.x-k8s.io_dockermachinetemplates.yaml

@@ -288,25 +288,31 @@ func (r *DockerMachineReconciler) reconcileNormal(ctx context.Context, cluster *
 
 	// if the machine isn't bootstrapped, only then run bootstrap scripts
 	if !dockerMachine.Spec.Bootstrapped {
-		bootstrapData, format, err := r.getBootstrapData(ctx, machine)
-		if err != nil {
-			log.Error(err, "failed to get bootstrap data")
-			return ctrl.Result{}, err
-		}
-
-		timeoutctx, cancel := context.WithTimeout(ctx, 3*time.Minute)
+		timeoutCtx, cancel := context.WithTimeout(ctx, 3*time.Minute)
 		defer cancel()
-		// Run the bootstrap script. Simulates cloud-init/Ignition.
-		if err := externalMachine.ExecBootstrap(timeoutctx, bootstrapData, format); err != nil {
-			conditions.MarkFalse(dockerMachine, infrav1.BootstrapExecSucceededCondition, infrav1.BootstrapFailedReason, clusterv1.ConditionSeverityWarning, "Repeating bootstrap")
-			return ctrl.Result{}, errors.Wrap(err, "failed to exec DockerMachine bootstrap")
-		}
+
 		// Check for bootstrap success
-		if err := externalMachine.CheckForBootstrapSuccess(timeoutctx); err != nil {
-			conditions.MarkFalse(dockerMachine, infrav1.BootstrapExecSucceededCondition, infrav1.BootstrapFailedReason, clusterv1.ConditionSeverityWarning, "Repeating bootstrap")
-			return ctrl.Result{}, errors.Wrap(err, "failed to check for existence of bootstrap success file at /run/cluster-api/bootstrap-success.complete")
-		}
+		// We have to check here to make this reentrant for cases where the bootstrap works
+		// but bootstrapped is never set on the object. We only try to bootstrap if the machine
+		// is not already bootstrapped.
+		if err := externalMachine.CheckForBootstrapSuccess(timeoutCtx, false); err != nil {
+			bootstrapData, format, err := r.getBootstrapData(timeoutCtx, machine)
+			if err != nil {
+				log.Error(err, "failed to get bootstrap data")
+				return ctrl.Result{}, err
+			}
 
+			// Run the bootstrap script. Simulates cloud-init/Ignition.
+			if err := externalMachine.ExecBootstrap(timeoutCtx, bootstrapData, format); err != nil {
+				conditions.MarkFalse(dockerMachine, infrav1.BootstrapExecSucceededCondition, infrav1.BootstrapFailedReason, clusterv1.ConditionSeverityWarning, "Repeating bootstrap")
+				return ctrl.Result{}, errors.Wrap(err, "failed to exec DockerMachine bootstrap")
+			}
+			// Check for bootstrap success
+			if err := externalMachine.CheckForBootstrapSuccess(timeoutCtx, true); err != nil {
+				conditions.MarkFalse(dockerMachine, infrav1.BootstrapExecSucceededCondition, infrav1.BootstrapFailedReason, clusterv1.ConditionSeverityWarning, "Repeating bootstrap")
+				return ctrl.Result{}, errors.Wrap(err, "failed to check for existence of bootstrap success file at /run/cluster-api/bootstrap-success.complete")
+			}
+		}
 		dockerMachine.Spec.Bootstrapped = true
 	}
 

test/infrastructure/docker/exp/api/v1beta1/dockermachinepool_types.go

@@ -355,7 +355,7 @@ func (m *Machine) ExecBootstrap(ctx context.Context, data string, format bootstr
 		}
 		err := cmd.Run(ctx)
 		if err != nil {
-			log.Info("Failed running command", "command", command, "stdout", outStd.String(), "stderr", outErr.String(), "bootstrap data", data)
+			log.Info("Failed running command", "instance", m.Name(), "command", command, "stdout", outStd.String(), "stderr", outErr.String(), "bootstrap data", data)
 			logContainerDebugInfo(ctx, log, m.ContainerName())
 			return errors.Wrap(errors.WithStack(err), "failed to run cloud config")
 		}
@@ -365,7 +365,7 @@ func (m *Machine) ExecBootstrap(ctx context.Context, data string, format bootstr
 }
 
 // CheckForBootstrapSuccess checks if bootstrap was successful by checking for existence of the sentinel file.
-func (m *Machine) CheckForBootstrapSuccess(ctx context.Context) error {
+func (m *Machine) CheckForBootstrapSuccess(ctx context.Context, logResult bool) error {
 	log := ctrl.LoggerFrom(ctx)
 
 	if m.container == nil {
@@ -378,7 +378,9 @@ func (m *Machine) CheckForBootstrapSuccess(ctx context.Context) error {
 	cmd.SetStderr(&outErr)
 	cmd.SetStdout(&outStd)
 	if err := cmd.Run(ctx); err != nil {
-		log.Info("Failed running command", "command", "test -f /run/cluster-api/bootstrap-success.complete", "stdout", outStd.String(), "stderr", outErr.String())
+		if logResult {
+			log.Info("Failed running command", "command", "test -f /run/cluster-api/bootstrap-success.complete", "stdout", outStd.String(), "stderr", outErr.String())
+		}
 		return errors.Wrap(errors.WithStack(err), "failed to run bootstrap check")
 	}
 	return nil