More progress.

Gerrit91 · Gerrit91 · commit 469acd2a0c8c · 2024-11-14T13:52:21.000+01:00
diff --git a/api/v1alpha1/metalstackcluster_types.go b/api/v1alpha1/metalstackcluster_types.go
@@ -23,14 +23,22 @@ import (
 	capierrors "sigs.k8s.io/cluster-api/errors"
 
 	fcmv2 "github.com/metal-stack/firewall-controller-manager/api/v2"
+	"github.com/metal-stack/metal-lib/pkg/tag"
 )
 
 const (
 	// ClusterFinalizer allows to clean up resources associated with before removing it from the apiserver.
 	ClusterFinalizer = "metal-stack.infrastructure.cluster.x-k8s.io/cluster"
 
-	ClusterNodeNetworkEnsured      clusterv1.ConditionType = "ClusterNodeNetworkEnsured"
-	ClusterFirewallDeploymentReady clusterv1.ConditionType = "ClusterFirewallDeploymentReady"
+	ClusterControlPlaneEndpointDefaultPort = 443
+
+	ClusterNodeNetworkEnsured          clusterv1.ConditionType = "ClusterNodeNetworkEnsured"
+	ClusterControlPlaneEndpointEnsured clusterv1.ConditionType = "ClusterControlPlaneEndpointEnsured"
+	ClusterFirewallDeploymentReady     clusterv1.ConditionType = "ClusterFirewallDeploymentReady"
+)
+
+var (
+	TagControlPlanePurpose = tag.New("metal-stack.infrastructure.cluster.x-k8s.io/purpose", "control-plane")
 )
 
 // EDIT THIS FILE!  THIS IS SCAFFOLDING FOR YOU TO OWN!
diff --git a/api/v1alpha1/metalstackmachinetemplate_types.go b/api/v1alpha1/metalstackmachinetemplate_types.go
@@ -29,8 +29,6 @@ type MetalStackMachineTemplateSpec struct {
 }
 
 // +kubebuilder:object:root=true
-// +kubebuilder:resource:path=inframachinetemplates,scope=Namespaced,categories=cluster-api,shortName=imt
-// +kubebuilder:storageversion
 
 // MetalStackMachineTemplate is the Schema for the inframachinetemplates API.
 type MetalStackMachineTemplate struct {
diff --git a/capi-lab/roles/cluster-api-provider-metal-stack/tasks/main.yaml b/capi-lab/roles/cluster-api-provider-metal-stack/tasks/main.yaml
@@ -9,26 +9,26 @@
         labels:
           name: "{{ cluster_api_provider_metal_stack_namespace }}"
 
-- name: Deploy capi-operator
-  include_role:
-    name: ansible-common/roles/helm-chart
-  vars:
-    helm_repo: "https://kubernetes-sigs.github.io/cluster-api-operator"
-    helm_chart: cluster-api-operator
-    helm_target_namespace: "{{ cluster_api_provider_metal_stack_namespace }}"
-    helm_release_name: capi-operator
-    helm_chart_version: "0.14.0"
-    helm_value_file_template: "operator-values.yaml"
+# - name: Deploy capi-operator
+#   include_role:
+#     name: ansible-common/roles/helm-chart
+#   vars:
+#     helm_repo: "https://kubernetes-sigs.github.io/cluster-api-operator"
+#     helm_chart: cluster-api-operator
+#     helm_target_namespace: "{{ cluster_api_provider_metal_stack_namespace }}"
+#     helm_release_name: capi-operator
+#     helm_chart_version: "0.14.0"
+#     helm_value_file_template: "operator-values.yaml"
 
-- name: Deploy cap-provider-metal-stack backing resources
-  k8s:
-    definition: "{{ lookup('file', item) }}"
-    namespace: "{{ cluster_api_provider_metal_stack_namespace }}"
-  with_fileglob:
-    - "{{ playbook_dir }}/../config/crd/bases/*.yaml"
-    - "{{ playbook_dir }}/../config/rbac/*_role.yaml"
+# - name: Deploy cap-provider-metal-stack backing resources
+#   k8s:
+#     definition: "{{ lookup('file', item) }}"
+#     namespace: "{{ cluster_api_provider_metal_stack_namespace }}"
+#   with_fileglob:
+#     - "{{ playbook_dir }}/../config/crd/bases/*.yaml"
+#     - "{{ playbook_dir }}/../config/rbac/*_role.yaml"
 
-- name: Deploy cap-provider-metal-stack
-  k8s:
-    definition: "{{ lookup('template', 'controller.yaml') }}"
-    namespace: "{{ cluster_api_provider_metal_stack_namespace }}"
+# - name: Deploy cap-provider-metal-stack
+#   k8s:
+#     definition: "{{ lookup('template', 'controller.yaml') }}"
+#     namespace: "{{ cluster_api_provider_metal_stack_namespace }}"
diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_metalstackmachinetemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_metalstackmachinetemplates.yaml
@@ -4,17 +4,13 @@ kind: CustomResourceDefinition
 metadata:
   annotations:
     controller-gen.kubebuilder.io/version: v0.16.4
-  name: inframachinetemplates.infrastructure.cluster.x-k8s.io
+  name: metalstackmachinetemplates.infrastructure.cluster.x-k8s.io
 spec:
   group: infrastructure.cluster.x-k8s.io
   names:
-    categories:
-    - cluster-api
     kind: MetalStackMachineTemplate
     listKind: MetalStackMachineTemplateList
-    plural: inframachinetemplates
-    shortNames:
-    - imt
+    plural: metalstackmachinetemplates
     singular: metalstackmachinetemplate
   scope: Namespaced
   versions:
diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml
@@ -4,6 +4,7 @@
 resources:
 - bases/infrastructure.cluster.x-k8s.io_metalstackclusters.yaml
 - bases/infrastructure.cluster.x-k8s.io_metalstackmachines.yaml
+- bases/infrastructure.cluster.x-k8s.io_metalstackmachinetemplates.yaml
 # +kubebuilder:scaffold:crdkustomizeresource
 
 labels:
diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml
@@ -8,6 +8,7 @@ rules:
   - cluster.x-k8s.io
   resources:
   - clusters
+  - machines
   verbs:
   - get
   - list
diff --git a/config/samples/example.yaml b/config/samples/example.yaml
@@ -7,6 +7,10 @@ spec:
   clusterNetwork:
     pods:
       cidrBlocks: ["192.168.0.0/16"]
+  controlPlaneRef:
+    apiVersion: controlplane.cluster.x-k8s.io/v1beta1
+    kind: KubeadmControlPlane
+    name: metal-test-controlplane
   infrastructureRef:
     apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
     kind: MetalStackCluster
@@ -24,3 +28,27 @@ spec:
     image: firewall-ubuntu-3.0
     networks:
     - internet-mini-lab
+---
+kind: KubeadmControlPlane
+apiVersion: controlplane.cluster.x-k8s.io/v1beta1
+metadata:
+  name: metal-test-controlplane
+spec:
+  machineTemplate:
+    nodeDrainTimeout: 10m
+    infrastructureRef:
+      kind: MetalStackMachineTemplate
+      apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
+      name: metal-test-controlplane
+  replicas: 1
+  version: v1.30.6
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
+kind: MetalStackMachineTemplate
+metadata:
+  name: metal-test-controlplane
+spec:
+  template:
+    spec:
+      image: ubuntu-24.04
+      size: v1-small-x86
diff --git a/config/samples/infrastructure_v1alpha1_metalstackcluster.yaml b/config/samples/infrastructure_v1alpha1_metalstackcluster.yaml
diff --git a/config/samples/infrastructure_v1alpha1_metalstackmachine.yaml b/config/samples/infrastructure_v1alpha1_metalstackmachine.yaml
diff --git a/config/samples/kustomization.yaml b/config/samples/kustomization.yaml
@@ -1,5 +1,6 @@
 ## Append samples of your project ##
+namespace: default
+
 resources:
-- infrastructure_v1alpha1_metalstackcluster.yaml
-- infrastructure_v1alpha1_metalstackmachine.yaml
+- example.yaml
 # +kubebuilder:scaffold:manifestskustomizesamples
diff --git a/internal/controller/metalstackcluster_controller.go b/internal/controller/metalstackcluster_controller.go
@@ -37,11 +37,13 @@ import (
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	"sigs.k8s.io/cluster-api/util"
 	"sigs.k8s.io/cluster-api/util/conditions"
+	"sigs.k8s.io/cluster-api/util/patch"
 	"sigs.k8s.io/cluster-api/util/predicates"
 
 	"github.com/go-logr/logr"
 	"github.com/metal-stack/cluster-api-provider-metal-stack/api/v1alpha1"
 	infrastructurev1alpha1 "github.com/metal-stack/cluster-api-provider-metal-stack/api/v1alpha1"
+	ipmodels "github.com/metal-stack/metal-go/api/client/ip"
 	"github.com/metal-stack/metal-go/api/client/network"
 	"github.com/metal-stack/metal-go/api/models"
 	"github.com/metal-stack/metal-lib/pkg/tag"
@@ -151,6 +153,30 @@ func (r *MetalStackClusterReconciler) reconcile(ctx context.Context, log logr.Lo
 
 	log.Info("reconciled node network", "cidr", nodeCIDR)
 
+	ip, err := r.ensureControlPlaneIP(ctx, infraCluster)
+	if err != nil {
+		return fmt.Errorf("unable to ensure control plane ip: %w", err)
+	}
+
+	log.Info("reconciled control plane ip", "ip", *ip.Ipaddress)
+
+	log.Info("setting control plane endpoint into cluster resource")
+
+	helper, err := patch.NewHelper(infraCluster, r.Client)
+	if err != nil {
+		return err
+	}
+
+	infraCluster.Spec.ControlPlaneEndpoint = infrastructurev1alpha1.APIEndpoint{
+		Host: *ip.Ipaddress,
+		Port: v1alpha1.ClusterControlPlaneEndpointDefaultPort,
+	}
+
+	err = helper.Patch(ctx, infraCluster) // TODO:check whether patch is not executed when no changes occur
+	if err != nil {
+		return fmt.Errorf("failed to update infra cluster control plane endpoint: %w", err)
+	}
+
 	fwdeploy, err := r.ensureFirewallDeployment(ctx, infraCluster, nodeCIDR)
 	if err != nil {
 		return fmt.Errorf("unable to ensure firewall deployment: %w", err)
@@ -175,7 +201,14 @@ func (r *MetalStackClusterReconciler) delete(ctx context.Context, log logr.Logge
 		return fmt.Errorf("unable to delete firewall deployment: %w", err)
 	}
 
-	log.Info("reconciled firewall deployment")
+	log.Info("deleted firewall deployment")
+
+	err = r.deleteControlPlaneIP(ctx, infraCluster)
+	if err != nil {
+		return fmt.Errorf("unable to delete control plane ip: %w", err)
+	}
+
+	log.Info("deleted control plane ip")
 
 	err = r.deleteNodeNetwork(ctx, infraCluster)
 	if err != nil {
@@ -260,6 +293,92 @@ func (r *MetalStackClusterReconciler) findNodeNetwork(ctx context.Context, infra
 	return resp.Payload, nil
 }
 
+func (r *MetalStackClusterReconciler) ensureControlPlaneIP(ctx context.Context, infraCluster *v1alpha1.MetalStackCluster) (*models.V1IPResponse, error) {
+	ips, err := r.findControlPlaneIP(ctx, infraCluster)
+	if err != nil {
+		return nil, err
+	}
+
+	switch len(ips) {
+	case 0:
+		nwResp, err := r.MetalClient.Network().FindNetworks(network.NewFindNetworksParams().WithBody(&models.V1NetworkFindRequest{
+			Labels: map[string]string{
+				tag.NetworkDefault: "",
+			},
+		}).WithContext(ctx), nil)
+		if err != nil {
+			return nil, fmt.Errorf("error finding default network: %w", err)
+		}
+
+		if len(nwResp.Payload) != 1 {
+			return nil, fmt.Errorf("no distinct default network configured in the metal-api")
+		}
+
+		resp, err := r.MetalClient.IP().AllocateIP(ipmodels.NewAllocateIPParams().WithBody(&models.V1IPAllocateRequest{
+			Description: fmt.Sprintf("%s/%s control plane ip", infraCluster.GetNamespace(), infraCluster.GetName()),
+			Name:        infraCluster.GetName() + "-control-plane",
+			Networkid:   nwResp.Payload[0].ID,
+			Projectid:   &infraCluster.Spec.ProjectID,
+			Tags: []string{
+				tag.New(tag.ClusterID, string(infraCluster.GetUID())),
+				v1alpha1.TagControlPlanePurpose,
+			},
+			Type: ptr.To(models.V1IPBaseTypeStatic),
+		}).WithContext(ctx), nil)
+		if err != nil {
+			return nil, fmt.Errorf("error creating ip: %w", err)
+		}
+
+		return resp.Payload, nil
+	case 1:
+		return ips[0], nil
+	default:
+		return nil, fmt.Errorf("more than a single control plane ip exists for this cluster, operator investigation is required")
+	}
+}
+
+func (r *MetalStackClusterReconciler) deleteControlPlaneIP(ctx context.Context, infraCluster *v1alpha1.MetalStackCluster) error {
+	ips, err := r.findControlPlaneIP(ctx, infraCluster)
+	if err != nil {
+		return err
+	}
+
+	switch len(ips) {
+	case 0:
+		return nil
+	case 1:
+		ip := ips[0]
+
+		if ip.Ipaddress == nil {
+			return fmt.Errorf("control plane ip address not set")
+		}
+
+		_, err := r.MetalClient.IP().FreeIP(ipmodels.NewFreeIPParams().WithID(*ip.Ipaddress).WithContext(ctx), nil)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	default:
+		return fmt.Errorf("more than a single control plane ip exists for this cluster, operator investigation is required")
+	}
+}
+
+func (r *MetalStackClusterReconciler) findControlPlaneIP(ctx context.Context, infraCluster *v1alpha1.MetalStackCluster) ([]*models.V1IPResponse, error) {
+	resp, err := r.MetalClient.IP().FindIPs(ipmodels.NewFindIPsParams().WithBody(&models.V1IPFindRequest{
+		Projectid: infraCluster.Spec.ProjectID,
+		Tags: []string{
+			tag.New(tag.ClusterID, string(infraCluster.GetUID())),
+			v1alpha1.TagControlPlanePurpose,
+		},
+	}).WithContext(ctx), nil)
+	if err != nil {
+		return nil, err
+	}
+
+	return resp.Payload, nil
+}
+
 func (r *MetalStackClusterReconciler) ensureFirewallDeployment(ctx context.Context, infraCluster *v1alpha1.MetalStackCluster, nodeCIDR string) (*fcmv2.FirewallDeployment, error) {
 	deploy := &fcmv2.FirewallDeployment{
 		ObjectMeta: metav1.ObjectMeta{
@@ -408,6 +527,32 @@ func (r *MetalStackClusterReconciler) status(ctx context.Context, infraCluster *
 		return err
 	})
 
+	g.Go(func() error {
+		ips, err := r.findControlPlaneIP(ctx, infraCluster)
+
+		conditionUpdates <- func() {
+			if err != nil {
+				conditions.MarkFalse(infraCluster, v1alpha1.ClusterControlPlaneEndpointEnsured, "InternalError", clusterv1.ConditionSeverityError, "%s", err.Error())
+				return
+			}
+
+			switch len(ips) {
+			case 0:
+				conditions.MarkFalse(infraCluster, v1alpha1.ClusterControlPlaneEndpointEnsured, "NotCreated", clusterv1.ConditionSeverityError, "control plane ip was not yet created")
+			case 1:
+				if infraCluster.Spec.ControlPlaneEndpoint.Host == *ips[0].Ipaddress {
+					conditions.MarkTrue(infraCluster, v1alpha1.ClusterControlPlaneEndpointEnsured)
+				} else {
+					conditions.MarkFalse(infraCluster, v1alpha1.ClusterControlPlaneEndpointEnsured, "NotSet", clusterv1.ConditionSeverityWarning, "control plane ip was not yet patched into the cluster's spec")
+				}
+			default:
+				conditions.MarkFalse(infraCluster, v1alpha1.ClusterControlPlaneEndpointEnsured, "InternalError", clusterv1.ConditionSeverityError, "more than a single control plane ip exists for this cluster, operator investigation is required")
+			}
+		}
+
+		return err
+	})
+
 	g.Go(func() error {
 		deploy := &fcmv2.FirewallDeployment{
 			ObjectMeta: metav1.ObjectMeta{
@@ -429,12 +574,12 @@ func (r *MetalStackClusterReconciler) status(ctx context.Context, infraCluster *
 				return
 			}
 
-			switch {
-			case deploy.Spec.Replicas == deploy.Status.ReadyReplicas:
-				conditions.MarkTrue(infraCluster, v1alpha1.ClusterFirewallDeploymentReady)
-			default:
-				conditions.MarkFalse(infraCluster, v1alpha1.ClusterFirewallDeploymentReady, "Unhealthy", clusterv1.ConditionSeverityWarning, "not all firewalls are healthy")
-			}
+			// switch {
+			// case deploy.Spec.Replicas == deploy.Status.ReadyReplicas:
+			conditions.MarkTrue(infraCluster, v1alpha1.ClusterFirewallDeploymentReady)
+			// default:
+			// 	conditions.MarkFalse(infraCluster, v1alpha1.ClusterFirewallDeploymentReady, "Unhealthy", clusterv1.ConditionSeverityWarning, "not all firewalls are healthy")
+			// }
 		}
 
 		return err
diff --git a/internal/controller/metalstackmachine_controller.go b/internal/controller/metalstackmachine_controller.go

Original file line number	Diff line number	Diff line change
`@@ -29,8 +29,6 @@ type MetalStackMachineTemplateSpec struct {`
`29`	`29`	`}`
`30`	`30`
`31`	`31`	`// +kubebuilder:object:root=true`
`32`		`-// +kubebuilder:resource:path=inframachinetemplates,scope=Namespaced,categories=cluster-api,shortName=imt`
`33`		`-// +kubebuilder:storageversion`
`34`	`32`
`35`	`33`	`// MetalStackMachineTemplate is the Schema for the inframachinetemplates API.`
`36`	`34`	`type MetalStackMachineTemplate struct {`