🌱 Hcloud: Provision hcloud machines with custom command (instead of Snapshots) (#1647)

Author: guettli

.mockery.yaml

@@ -1,4 +1,4 @@
-# This config was choosen, so that the output matches to old structure (pre config file .mockery.yaml).
+# This config was chosen, so that the output matches to old structure (pre config file .mockery.yaml).
 # If you are here to copy this config to a new project, then it might
 # make sense to choose a structure which needs less config by using
 # the default values of Mockery.

api/v1beta1/hcloudmachine_types.go

@@ -46,7 +46,33 @@ type HCloudMachineSpec struct {
 	// ImageName is the reference to the Machine Image from which to create the machine instance.
 	// It can reference an image uploaded to Hetzner API in two ways: either directly as the name of an image or as the label of an image.
 	// +kubebuilder:validation:MinLength=1
-	ImageName string `json:"imageName"`
+	// +kubebuilder:validation:Optional
+	// +optional
+	ImageName string `json:"imageName,omitempty"`
+
+	// ImageURL gets used for installing custom node images. If that field is set, the controller
+	// boots a new HCloud machine into rescue mode. Then the script provided by
+	// --hcloud-image-url-command (which you need to provide to the controller binary) will be
+	// copied into the rescue system and executed.
+	//
+	// The controller uses url.ParseRequestURI (Go function) to validate the URL.
+	//
+	// It is up to the script to provision the disk of the hcloud machine accordingly. The process
+	// is considered successful if the last line in the output contains
+	// IMAGE_URL_DONE. If the script terminates with a different last line, then
+	// the process is considered to have failed.
+	//
+	// A Kubernetes event will be created in both (success, failure) cases containing the output
+	// (stdout and stderr) of the script. If the script takes longer than 7 minutes, the
+	// controller cancels the provisioning.
+	//
+	// Docs: https://syself.com/docs/caph/developers/image-url-command
+	//
+	// ImageURL is mutually exclusive to "ImageName".
+	// +kubebuilder:validation:MinLength=1
+	// +kubebuilder:validation:Optional
+	// +optional
+	ImageURL string `json:"imageURL,omitempty"`
 
 	// SSHKeys define machine-specific SSH keys and override cluster-wide SSH keys.
 	// +optional
@@ -99,16 +125,35 @@ type HCloudMachineStatus struct {
 
 	// BootState indicates the current state during provisioning.
 	//
-	// The states will be:
-	// "" -> BootToRealOS -> OperatingSystemRunning
+	// If Spec.ImageName is set the states will be:
+	//   1. BootingToRealOS
+	//   2. OperatingSystemRunning
 	//
+	// If Spec.ImageURL is set the states will be:
+	//   1. Initializing
+	//   2. EnablingRescue
+	//   3. BootingToRescue
+	//   4. RunningImageCommand
+	//   5. BootingToRealOS
+	//   6. OperatingSystemRunning
+
 	// +optional
 	BootState HCloudBootState `json:"bootState"`
 
 	// BootStateSince is the timestamp of the last change to BootState. It is used to timeout
 	// provisioning if a state takes too long.
 	// +optional
 	BootStateSince metav1.Time `json:"bootStateSince,omitzero"`
+
+	// ExternalIDs contains temporary data during the provisioning process
+	ExternalIDs HCloudMachineStatusExternalIDs `json:"externalIDs,omitempty"`
+}
+
+// HCloudMachineStatusExternalIDs holds temporary data during the provisioning process.
+type HCloudMachineStatusExternalIDs struct {
+	// ActionIDEnableRescueSystem is the hcloud API Action result of EnableRescueSystem.
+	// +optional
+	ActionIDEnableRescueSystem int64 `json:"actionIdEnableRescueSystem,omitzero"`
 }
 
 // HCloudMachine is the Schema for the hcloudmachines API.

api/v1beta1/hcloudmachine_validation.go

@@ -17,6 +17,7 @@ limitations under the License.
 package v1beta1
 
 import (
+	"net/url"
 	"reflect"
 
 	"k8s.io/apimachinery/pkg/util/validation/field"
@@ -38,6 +39,13 @@ func validateHCloudMachineSpecUpdate(oldSpec, newSpec HCloudMachineSpec) field.E
 		)
 	}
 
+	// ImageURL is immutable
+	if !reflect.DeepEqual(oldSpec.ImageURL, newSpec.ImageURL) {
+		allErrs = append(allErrs,
+			field.Invalid(field.NewPath("spec", "imageURL"), newSpec.ImageURL, "field is immutable"),
+		)
+	}
+
 	// SSHKeys is immutable
 	if !reflect.DeepEqual(oldSpec.SSHKeys, newSpec.SSHKeys) {
 		allErrs = append(allErrs,
@@ -52,5 +60,30 @@ func validateHCloudMachineSpecUpdate(oldSpec, newSpec HCloudMachineSpec) field.E
 		)
 	}
 
+	allErrs = append(allErrs, validateHCloudMachineSpec(newSpec)...)
+
+	return allErrs
+}
+
+func validateHCloudMachineSpec(spec HCloudMachineSpec) field.ErrorList {
+	var allErrs field.ErrorList
+	if spec.ImageName != "" && spec.ImageURL != "" {
+		allErrs = append(allErrs,
+			field.Invalid(field.NewPath("spec", "imageName"), spec.ImageName, "imageName and imageURL are mutually exclusive"))
+	}
+
+	if spec.ImageName == "" && spec.ImageURL == "" {
+		allErrs = append(allErrs,
+			field.Invalid(field.NewPath("spec", "imageName"), spec.ImageName, "imageName and imageURL empty. One of these attributes must be set"))
+	}
+
+	if spec.ImageURL != "" {
+		_, err := url.ParseRequestURI(spec.ImageURL)
+		if err != nil {
+			allErrs = append(allErrs,
+				field.Invalid(field.NewPath("spec", "imageURL"), spec.ImageURL, err.Error()))
+		}
+	}
+
 	return allErrs
 }

api/v1beta1/hcloudmachine_validation_test.go

@@ -17,9 +17,11 @@ limitations under the License.
 package v1beta1
 
 import (
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 	"k8s.io/apimachinery/pkg/util/validation/field"
 )
 
@@ -60,6 +62,18 @@ func TestValidateHCloudMachineSpecUpdate(t *testing.T) {
 			},
 			want: field.Invalid(field.NewPath("spec", "imageName"), "centos-7", "field is immutable"),
 		},
+		{
+			name: "Immutable ImageURL",
+			args: args{
+				oldSpec: HCloudMachineSpec{
+					ImageURL: "oci://ghcr.io/example/foo:v1",
+				},
+				newSpec: HCloudMachineSpec{
+					ImageURL: "oci://ghcr.io/example/foo:v2",
+				},
+			},
+			want: field.Invalid(field.NewPath("spec", "imageURL"), "oci://ghcr.io/example/foo:v2", "field is immutable"),
+		},
 		{
 			name: "Immutable SSHKeys",
 			args: args{
@@ -152,3 +166,24 @@ func TestValidateHCloudMachineSpecUpdate(t *testing.T) {
 func createPlacementGroupName(name string) *string {
 	return &name
 }
+
+func TestValidateHCloudMachineSpec(t *testing.T) {
+	allErrs := validateHCloudMachineSpec(HCloudMachineSpec{
+		ImageURL: "not-a-valid-url",
+	})
+	require.Equal(t, `spec.imageURL: Invalid value: "not-a-valid-url": parse "not-a-valid-url": invalid URI for request`, errorsToString(allErrs))
+
+	allErrs = validateHCloudMachineSpec(HCloudMachineSpec{
+		ImageName: "foo-name",
+		ImageURL:  "oci://ghcr.io/example/foo:v1",
+	})
+	require.Equal(t, `spec.imageName: Invalid value: "foo-name": imageName and imageURL are mutually exclusive`, errorsToString(allErrs))
+}
+
+func errorsToString(allErrs field.ErrorList) string {
+	s := make([]string, 0, len(allErrs))
+	for _, err := range allErrs {
+		s = append(s, err.Error())
+	}
+	return strings.Join(s, "\n")
+}

api/v1beta1/hcloudmachine_webhook.go

@@ -22,7 +22,6 @@ import (
 
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/apimachinery/pkg/util/validation/field"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/webhook"
 	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
@@ -83,7 +82,8 @@ func (*hcloudMachineWebhook) ValidateCreate(_ context.Context, obj runtime.Objec
 	}
 
 	hcloudmachinelog.V(1).Info("validate create", "name", r.Name)
-	var allErrs field.ErrorList
+
+	allErrs := validateHCloudMachineSpec(r.Spec)
 
 	return nil, aggregateObjErrors(r.GroupVersionKind().GroupKind(), r.Name, allErrs)
 }

api/v1beta1/types.go

@@ -271,8 +271,24 @@ const (
 	// HCloudBootStateUnset is the initial state when the boot state has not been set yet.
 	HCloudBootStateUnset HCloudBootState = ""
 
-	// HCloudBootStateBootToRealOS indicates that the server is booting the operating system.
-	HCloudBootStateBootToRealOS HCloudBootState = "BootToRealOS"
+	// HCloudBootStateInitializing indicates that the controller waits for PreRescueOS.
+	// When it is available, then the rescue system gets enabled.
+	HCloudBootStateInitializing HCloudBootState = "Initializing"
+
+	// HCloudBootStateEnablingRescue indicates that the controller waits for the rescue system to be enabled. Then the server gets booted into the rescue system.
+	HCloudBootStateEnablingRescue HCloudBootState = "EnablingRescue"
+
+	// HCloudBootStateBootingToRescue indicates that the controller
+	// waits for the rescue system to be reachable. Then it starts the image-url-command.
+	HCloudBootStateBootingToRescue HCloudBootState = "BootingToRescue"
+
+	// HCloudBootStateRunningImageCommand indicates the controller waits for the
+	// image-url-command, and then switches BootState to BootingToRealOS (no additional reboot gets
+	// done).
+	HCloudBootStateRunningImageCommand HCloudBootState = "RunningImageCommand"
+
+	// HCloudBootStateBootingToRealOS indicates that the server is booting the operating system.
+	HCloudBootStateBootingToRealOS HCloudBootState = "BootingToRealOS"
 
 	// HCloudBootStateOperatingSystemRunning indicates that the server is successfully running.
 	HCloudBootStateOperatingSystemRunning HCloudBootState = "OperatingSystemRunning"

api/v1beta1/zz_generated.deepcopy.go

@@ -72,6 +72,29 @@ spec:
                   It can reference an image uploaded to Hetzner API in two ways: either directly as the name of an image or as the label of an image.
                 minLength: 1
                 type: string
+              imageURL:
+                description: |-
+                  ImageURL gets used for installing custom node images. If that field is set, the controller
+                  boots a new HCloud machine into rescue mode. Then the script provided by
+                  --hcloud-image-url-command (which you need to provide to the controller binary) will be
+                  copied into the rescue system and executed.
+
+                  The controller uses url.ParseRequestURI (Go function) to validate the URL.
+
+                  It is up to the script to provision the disk of the hcloud machine accordingly. The process
+                  is considered successful if the last line in the output contains
+                  IMAGE_URL_DONE. If the script terminates with a different last line, then
+                  the process is considered to have failed.
+
+                  A Kubernetes event will be created in both (success, failure) cases containing the output
+                  (stdout and stderr) of the script. If the script takes longer than 7 minutes, the
+                  controller cancels the provisioning.
+
+                  Docs: https://syself.com/docs/caph/developers/image-url-command
+
+                  ImageURL is mutually exclusive to "ImageName".
+                minLength: 1
+                type: string
               placementGroupName:
                 description: PlacementGroupName defines the placement group of the
                   machine in HCloud API that must reference an existing placement
@@ -157,7 +180,6 @@ spec:
                 - cx52
                 type: string
             required:
-            - imageName
             - type
             type: object
           status:
@@ -190,11 +212,7 @@ spec:
                   type: object
                 type: array
               bootState:
-                description: |-
-                  BootState indicates the current state during provisioning.
-
-                  The states will be:
-                  "" -> BootToRealOS -> OperatingSystemRunning
+                description: HCloudBootState defines the boot state of an HCloud server.
                 type: string
               bootStateSince:
                 description: |-
@@ -254,6 +272,16 @@ spec:
                   - type
                   type: object
                 type: array
+              externalIDs:
+                description: ExternalIDs contains temporary data during the provisioning
+                  process
+                properties:
+                  actionIdEnableRescueSystem:
+                    description: ActionIDEnableRescueSystem is the hcloud API Action
+                      result of EnableRescueSystem.
+                    format: int64
+                    type: integer
+                type: object
               failureMessage:
                 description: |-
                   FailureMessage will be set in the event that there is a terminal problem

config/crd/bases/infrastructure.cluster.x-k8s.io_hcloudmachines.yaml

@@ -99,6 +99,29 @@ spec:
                           It can reference an image uploaded to Hetzner API in two ways: either directly as the name of an image or as the label of an image.
                         minLength: 1
                         type: string
+                      imageURL:
+                        description: |-
+                          ImageURL gets used for installing custom node images. If that field is set, the controller
+                          boots a new HCloud machine into rescue mode. Then the script provided by
+                          --hcloud-image-url-command (which you need to provide to the controller binary) will be
+                          copied into the rescue system and executed.
+
+                          The controller uses url.ParseRequestURI (Go function) to validate the URL.
+
+                          It is up to the script to provision the disk of the hcloud machine accordingly. The process
+                          is considered successful if the last line in the output contains
+                          IMAGE_URL_DONE. If the script terminates with a different last line, then
+                          the process is considered to have failed.
+
+                          A Kubernetes event will be created in both (success, failure) cases containing the output
+                          (stdout and stderr) of the script. If the script takes longer than 7 minutes, the
+                          controller cancels the provisioning.
+
+                          Docs: https://syself.com/docs/caph/developers/image-url-command
+
+                          ImageURL is mutually exclusive to "ImageName".
+                        minLength: 1
+                        type: string
                       placementGroupName:
                         description: PlacementGroupName defines the placement group
                           of the machine in HCloud API that must reference an existing
@@ -184,7 +207,6 @@ spec:
                         - cx52
                         type: string
                     required:
-                    - imageName
                     - type
                     type: object
                 required:

config/crd/bases/infrastructure.cluster.x-k8s.io_hcloudmachinetemplates.yaml

@@ -83,6 +83,7 @@ var _ = BeforeSuite(func() {
 		Client:              testEnv.Manager.GetClient(),
 		APIReader:           testEnv.Manager.GetAPIReader(),
 		HCloudClientFactory: testEnv.HCloudClientFactory,
+		SSHClientFactory:    testEnv.BaremetalSSHClientFactory,
 	}).SetupWithManager(ctx, testEnv.Manager, controller.Options{})).To(Succeed())
 
 	Expect((&HCloudMachineTemplateReconciler{
@@ -95,7 +96,7 @@ var _ = BeforeSuite(func() {
 		Client:              testEnv.Manager.GetClient(),
 		APIReader:           testEnv.Manager.GetAPIReader(),
 		RobotClientFactory:  testEnv.RobotClientFactory,
-		SSHClientFactory:    testEnv.SSHClientFactory,
+		SSHClientFactory:    testEnv.BaremetalSSHClientFactory,
 		PreProvisionCommand: "dummy-pre-provision-command",
 	}).SetupWithManager(ctx, testEnv.Manager, controller.Options{})).To(Succeed())