feat: get rid of init nodes and use bootstrap API to setup cluster

That required couple changes in the manager flow:
- no longer get node IPs using `kubectl get nodes`, but use machines
addresses instead.
- add `MachinesBootstrapped` condition that means that `bootstrap` was
called on one of the machines.
- add `Bootstrapped` flag to the `TalosControlPlaneStatus` to track
bootstrap call.

Signed-off-by: Artem Chernyshev <[email protected]>

Author: Unix4ever

.drone.yml

@@ -81,6 +81,8 @@ steps:
         from_secret: aws_secret_access_key
       CI: true
       REGISTRY: registry.dev.talos-systems.io
+      GITHUB_TOKEN:
+        from_secret: github_token
     commands:
       - make integration-test
     volumes:

api/v1alpha3/conditions.go

@@ -15,6 +15,15 @@ const (
 	MachinesReadyCondition clusterv1.ConditionType = "MachinesReady"
 )
 
+const (
+	// MachinesBootstrapped is tracking control planes bootstrap status.
+	MachinesBootstrapped clusterv1.ConditionType = "MachinesBootstrapped"
+
+	// WaitingForMachinesReason (Severity=Info) documents a TalosControlPlane bootstrap is waiting
+	// for all control plane nodes to be created.
+	WaitingForMachinesReason = "WaitingForMachines"
+)
+
 const (
 	// AvailableCondition documents that the first control plane instance has completed Talos boot sequence
 	// and so the control plane is available and an API server instance is ready for processing requests.
@@ -23,6 +32,10 @@ const (
 	// WaitingForTalosBootReason (Severity=Info) documents a TalosControlPlane object waiting for the first
 	// control plane instance to complete Talos boot sequence.
 	WaitingForTalosBootReason = "WaitingForTalosBoot"
+
+	// InvalidControlPlaneConfigReason (Severity=Error) documents that controlplane config is invalid and the provider
+	// can not proceed with the bootstrap.
+	InvalidControlPlaneConfigReason = "InvalidControlPlaneConfig"
 )
 
 const (

api/v1alpha3/taloscontrolplane_types.go

@@ -16,7 +16,8 @@ const (
 )
 
 type ControlPlaneConfig struct {
-	InitConfig         cabptv1.TalosConfigSpec `json:"init"`
+	// Deprecated: starting from cacppt v0.4.0 provider doesn't use init configs.
+	InitConfig         cabptv1.TalosConfigSpec `json:"init,omitempty"`
 	ControlPlaneConfig cabptv1.TalosConfigSpec `json:"controlplane"`
 }
 
@@ -79,6 +80,11 @@ type TalosControlPlaneStatus struct {
 	// +optional
 	Ready bool `json:"ready"`
 
+	// Bootstrapped denotes whether any nodes received bootstrap request
+	// which is required to start etcd and Kubernetes components in Talos.
+	// +optional
+	Bootstrapped bool `json:"bootstrapped,omitempty"`
+
 	// FailureReason indicates that there is a terminal problem reconciling the
 	// state, and will be set to a token value suitable for
 	// programmatic interpretation.

config/crd/bases/controlplane.cluster.x-k8s.io_taloscontrolplanes.yaml

@@ -87,7 +87,7 @@ spec:
                     - generateType
                     type: object
                   init:
-                    description: TalosConfigSpec defines the desired state of TalosConfig
+                    description: 'Deprecated: starting from cacppt v0.4.0 provider doesn''t use init configs.'
                     properties:
                       configPatches:
                         items:
@@ -114,7 +114,6 @@ spec:
                     type: object
                 required:
                 - controlplane
-                - init
                 type: object
               infrastructureTemplate:
                 description: InfrastructureTemplate is a required reference to a custom resource offered by an infrastructure provider.
@@ -158,6 +157,9 @@ spec:
           status:
             description: TalosControlPlaneStatus defines the observed state of TalosControlPlane
             properties:
+              bootstrapped:
+                description: Bootstrapped denotes wheither any nodes recieved bootstrap request which is required to start etcd and Kubernetes components in Talos.
+                type: boolean
               conditions:
                 description: Conditions defines current service state of the KubeadmControlPlane.
                 items:

controllers/configs.go

@@ -14,7 +14,6 @@ import (
 	talosclient "github.com/talos-systems/talos/pkg/machinery/client"
 	talosconfig "github.com/talos-systems/talos/pkg/machinery/client/config"
 	corev1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/clientcmd"
@@ -76,7 +75,7 @@ func (r *TalosControlPlaneReconciler) kubeconfigForCluster(ctx context.Context,
 }
 
 // talosconfigForMachine will generate a talosconfig that uses *all* found addresses as the endpoints.
-func (r *TalosControlPlaneReconciler) talosconfigForMachines(ctx context.Context, clientset *kubernetes.Clientset, machines ...clusterv1.Machine) (*talosclient.Client, error) {
+func (r *TalosControlPlaneReconciler) talosconfigForMachines(ctx context.Context, machines ...clusterv1.Machine) (*talosclient.Client, error) {
 	if len(machines) == 0 {
 		return nil, fmt.Errorf("at least one machine should be provided")
 	}
@@ -86,24 +85,14 @@ func (r *TalosControlPlaneReconciler) talosconfigForMachines(ctx context.Context
 	var t *talosconfig.Config
 
 	for _, machine := range machines {
-		if machine.Status.NodeRef == nil {
-			return nil, fmt.Errorf("%q machine does not have a nodeRef", machine.Name)
-		}
-
-		// grab all addresses as endpoints
-		node, err := clientset.CoreV1().Nodes().Get(ctx, machine.Status.NodeRef.Name, metav1.GetOptions{})
-		if err != nil {
-			return nil, err
-		}
-
-		for _, addr := range node.Status.Addresses {
-			if addr.Type == corev1.NodeExternalIP || addr.Type == corev1.NodeInternalIP {
+		for _, addr := range machine.Status.Addresses {
+			if addr.Type == clusterv1.MachineExternalIP || addr.Type == clusterv1.MachineInternalIP {
 				addrList = append(addrList, addr.Address)
 			}
 		}
 
 		if len(addrList) == 0 {
-			return nil, fmt.Errorf("no addresses were found for node %q", node.Name)
+			return nil, fmt.Errorf("no addresses were found for node %q", machine.Name)
 		}
 
 		if t == nil {
@@ -113,7 +102,7 @@ func (r *TalosControlPlaneReconciler) talosconfigForMachines(ctx context.Context
 			)
 
 			// find talosconfig in the machine's namespace
-			err = r.Client.List(ctx, &cfgs, client.InNamespace(machine.Namespace))
+			err := r.Client.List(ctx, &cfgs, client.InNamespace(machine.Namespace))
 			if err != nil {
 				return nil, err
 			}

controllers/consts.go

@@ -0,0 +1,21 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package controllers
+
+import "time"
+
+const (
+	// deleteRequeueAfter is how long to wait before checking again to see if
+	// all control plane machines have been deleted.
+	deleteRequeueAfter = 30 * time.Second
+
+	// preflightFailedRequeueAfter is how long to wait before trying to scale
+	// up/down if some preflight check for those operation has failed.
+	preflightFailedRequeueAfter = 15 * time.Second
+
+	// dependentCertRequeueAfter is how long to wait before checking again to see if
+	// dependent certificates have been created.
+	dependentCertRequeueAfter = 30 * time.Second
+)

controllers/etcd.go

@@ -32,7 +32,7 @@ func (r *TalosControlPlaneReconciler) etcdHealthcheck(ctx context.Context, clust
 		}
 	}
 
-	c, err := r.talosconfigForMachines(ctx, kubeclient.Clientset, machines...)
+	c, err := r.talosconfigForMachines(ctx, machines...)
 	if err != nil {
 		return err
 	}
@@ -182,14 +182,7 @@ func (r *TalosControlPlaneReconciler) auditEtcd(ctx context.Context, cluster cli
 		return fmt.Errorf("no CP machine which is not being deleted and has node ref")
 	}
 
-	kubeclient, err := r.kubeconfigForCluster(ctx, cluster)
-	if err != nil {
-		return err
-	}
-
-	defer kubeclient.Close() //nolint:errcheck
-
-	c, err := r.talosconfigForMachines(ctx, kubeclient.Clientset, designatedCPMachine)
+	c, err := r.talosconfigForMachines(ctx, designatedCPMachine)
 	if err != nil {
 		return err
 	}

controllers/health.go

@@ -16,7 +16,6 @@ import (
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
-	"sigs.k8s.io/cluster-api/util"
 )
 
 type errServiceUnhealthy struct {
@@ -29,14 +28,7 @@ func (e *errServiceUnhealthy) Error() string {
 }
 
 func (r *TalosControlPlaneReconciler) nodesHealthcheck(ctx context.Context, cluster *clusterv1.Cluster, machines []clusterv1.Machine) error {
-	kubeclient, err := r.kubeconfigForCluster(ctx, util.ObjectKey(cluster))
-	if err != nil {
-		return err
-	}
-
-	defer kubeclient.Close() //nolint:errcheck
-
-	client, err := r.talosconfigForMachines(ctx, kubeclient.Clientset, machines...)
+	client, err := r.talosconfigForMachines(ctx, machines...)
 	if err != nil {
 		return err
 	}
@@ -63,14 +55,7 @@ func (r *TalosControlPlaneReconciler) nodesHealthcheck(ctx context.Context, clus
 }
 
 func (r *TalosControlPlaneReconciler) ensureNodesBooted(ctx context.Context, cluster *clusterv1.Cluster, machines []clusterv1.Machine) error {
-	kubeclient, err := r.kubeconfigForCluster(ctx, util.ObjectKey(cluster))
-	if err != nil {
-		return err
-	}
-
-	defer kubeclient.Close() //nolint:errcheck
-
-	client, err := r.talosconfigForMachines(ctx, kubeclient.Clientset, machines...)
+	client, err := r.talosconfigForMachines(ctx, machines...)
 	if err != nil {
 		return err
 	}