feat: generate talosconfig as a secret with proper endpoints

Fixes #83 fixes #85

Generate cluster-wide `talosconfig` in a secret `<cluster>-talosconfig`.
Reconcile controlplane machine addresses and put them as `endpoints:` in
the generated `talosconfig`.

Other fixes:

* use proper method to generate `talosconfig`
* generate regular `talosconfig` in Status for user-supplied machine config if possible
* handle pause for cluster reconcilation

Signed-off-by: Andrey Smirnov <[email protected]>

Author: smira

Makefile

@@ -7,6 +7,7 @@ REGISTRY_AND_USERNAME := $(REGISTRY)/$(USERNAME)
 NAME := cluster-api-talos-controller
 
 ARTIFACTS := _out
+TEST_RUN ?= ./...
 
 TOOLS ?= ghcr.io/talos-systems/tools:v0.8.0-alpha.0-3-g2790b55
 PKGS ?= v0.8.0
@@ -130,7 +131,7 @@ conformance:  ## Performs policy checks against the commit and source code.
 # Make `make test` behave just like `go test` regarding relative paths.
 test:  ## Run tests.
 	@$(MAKE) local-integration-test DEST=./internal/integration PLATFORM=linux/amd64
-	cd internal/integration && KUBECONFIG=../../kubeconfig ./integration.test -test.v -test.coverprofile=../../coverage.txt
+	cd internal/integration && KUBECONFIG=../../kubeconfig ./integration.test -test.v -test.coverprofile=../../coverage.txt -test.run $(TEST_RUN)
 
 coverage:  ## Upload coverage data to codecov.io.
 	/usr/local/bin/codecov -f coverage.txt -X fix

api/v1alpha3/conditions.go

@@ -30,3 +30,13 @@ const (
 	// and user intervention is required to get them fixed.
 	DataSecretGenerationFailedReason = "DataSecretGenerationFailed"
 )
+
+const (
+	// ClientConfigAvailableCondition documents the status of the client config generation process.
+	ClientConfigAvailableCondition capiv1.ConditionType = "ClientConfigAvailable"
+
+	// ClientConfigGenerationFailedReason (Severity=Warning) documents a TalosConfig controller detecting
+	// an error while generating a client config; those kind of errors are usually due to misconfigurations
+	// and user intervention is required to get them fixed.
+	ClientConfigGenerationFailedReason = "ClientConfigGenerationFailed"
+)

api/v1alpha3/talosconfig_types.go

@@ -32,7 +32,10 @@ type TalosConfigStatus struct {
 	// +optional
 	DataSecretName *string `json:"dataSecretName,omitempty"`
 
-	// Talos config will be a string containing the config for download
+	// Talos config will be a string containing the config for download.
+	//
+	// Deprecated: please use `<cluster>-talosconfig` secret.
+	//
 	// +optional
 	TalosConfig string `json:"talosConfig,omitempty"`
 

controllers/secrets.go

@@ -7,14 +7,18 @@ package controllers
 import (
 	"context"
 	"fmt"
+	"sort"
 
+	"github.com/go-logr/logr"
 	"github.com/talos-systems/crypto/x509"
 	"github.com/talos-systems/talos/pkg/machinery/config/types/v1alpha1/generate"
+	talosmachine "github.com/talos-systems/talos/pkg/machinery/config/types/v1alpha1/machine"
 	"gopkg.in/yaml.v2"
 	corev1 "k8s.io/api/core/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	capiv1 "sigs.k8s.io/cluster-api/api/v1alpha4"
+	"sigs.k8s.io/cluster-api/util/collections"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	bootstrapv1alpha3 "github.com/talos-systems/cluster-api-bootstrap-provider-talos/api/v1alpha3"
@@ -35,14 +39,20 @@ func (r *TalosConfigReconciler) fetchSecret(ctx context.Context, config *bootstr
 }
 
 // getSecretsBundle either generates or loads existing secret.
-func (r *TalosConfigReconciler) getSecretsBundle(ctx context.Context, scope *TalosConfigScope, secretName string, opts ...generate.GenOption) (*generate.SecretsBundle, error) {
+func (r *TalosConfigReconciler) getSecretsBundle(ctx context.Context, scope *TalosConfigScope, allowGenerate bool, opts ...generate.GenOption) (*generate.SecretsBundle, error) {
 	var secretsBundle *generate.SecretsBundle
 
+	secretName := scope.Cluster.Name + "-talos"
+
 retry:
 	secret, err := r.fetchSecret(ctx, scope.Config, secretName)
 
 	switch {
 	case err != nil && k8serrors.IsNotFound(err):
+		if !allowGenerate {
+			return nil, fmt.Errorf("secrets bundle is missing")
+		}
+
 		// no cluster secret yet, generate new one
 		secretsBundle, err = generate.NewSecretsBundle(generate.NewClock(), opts...)
 		if err != nil {
@@ -190,3 +200,80 @@ func (r *TalosConfigReconciler) writeBootstrapData(ctx context.Context, scope *T
 
 	return dataSecretName, err
 }
+
+// reconcileClientConfig creates/updates a TalosConfig for the cluster.
+func (r *TalosConfigReconciler) reconcileClientConfig(ctx context.Context, log logr.Logger, scope *TalosConfigScope) error {
+	if !(scope.Config.Spec.GenerateType == talosmachine.TypeControlPlane.String() || scope.Config.Spec.GenerateType == talosmachine.TypeInit.String()) {
+		// can only reconcile for control plane machines
+		return nil
+	}
+
+	machines, err := collections.GetFilteredMachinesForCluster(ctx, r.Client, scope.Cluster, collections.ControlPlaneMachines(scope.Cluster.Name))
+	if err != nil {
+		return fmt.Errorf("failed getting control plane machines: %w", err)
+	}
+
+	var endpoints []string
+
+	for _, machine := range machines {
+		for _, addr := range machine.Status.Addresses {
+			if addr.Type == capiv1.MachineExternalIP || addr.Type == capiv1.MachineInternalIP {
+				endpoints = append(endpoints, addr.Address)
+			}
+		}
+	}
+
+	sort.Strings(endpoints)
+
+	secretBundle, err := r.getSecretsBundle(ctx, scope, false)
+	if err != nil {
+		return err
+	}
+
+	talosConfig, err := genTalosConfigFile(scope.Cluster.Name, secretBundle, endpoints)
+	if err != nil {
+		return err
+	}
+
+	// Create or update talosconfig secret
+	dataSecretName := scope.Cluster.GetName() + "-talosconfig"
+
+	log.Info("updating talosconfig", "endpoints", endpoints, "secret", dataSecretName)
+
+	configSecret := &corev1.Secret{}
+
+	err = r.Client.Get(ctx, client.ObjectKey{Namespace: scope.Cluster.Namespace, Name: dataSecretName}, configSecret)
+	if err != nil {
+		if !k8serrors.IsNotFound(err) {
+			return fmt.Errorf("error fetching secret: %w", err)
+		}
+
+		configSecret = &corev1.Secret{
+			ObjectMeta: metav1.ObjectMeta{
+				Namespace: scope.Cluster.Namespace,
+				Name:      dataSecretName,
+				Labels: map[string]string{
+					capiv1.ClusterLabelName: scope.Cluster.Name,
+				},
+				OwnerReferences: []metav1.OwnerReference{
+					*metav1.NewControllerRef(scope.Cluster, capiv1.GroupVersion.WithKind("Cluster")),
+				},
+			},
+			Data: map[string][]byte{
+				"talosconfig": []byte(talosConfig),
+			},
+		}
+
+		return r.Client.Create(ctx, configSecret)
+	}
+
+	configSecret.Data["talosconfig"] = []byte(talosConfig)
+
+	err = r.Client.Update(ctx, configSecret)
+	if k8serrors.IsConflict(err) {
+		// ignore conflict errors, probably another reconcile fixed up the endpoints
+		err = nil
+	}
+
+	return err
+}

controllers/talosconfig_controller.go

@@ -6,12 +6,12 @@ package controllers
 
 import (
 	"context"
-	"encoding/base64"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"strconv"
 	"strings"
+	"time"
 
 	jsonpatch "github.com/evanphx/json-patch"
 	"github.com/go-logr/logr"
@@ -21,6 +21,7 @@ import (
 	"github.com/talos-systems/talos/pkg/machinery/config/types/v1alpha1/generate"
 	"github.com/talos-systems/talos/pkg/machinery/config/types/v1alpha1/machine"
 	"github.com/talos-systems/talos/pkg/machinery/constants"
+	"github.com/talos-systems/talos/pkg/machinery/role"
 	"gopkg.in/yaml.v2"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -29,6 +30,7 @@ import (
 	expv1 "sigs.k8s.io/cluster-api/exp/api/v1alpha4"
 	"sigs.k8s.io/cluster-api/feature"
 	"sigs.k8s.io/cluster-api/util"
+	"sigs.k8s.io/cluster-api/util/annotations"
 	"sigs.k8s.io/cluster-api/util/conditions"
 	"sigs.k8s.io/cluster-api/util/patch"
 	"sigs.k8s.io/cluster-api/util/predicates"
@@ -70,18 +72,6 @@ type TalosConfigBundle struct {
 	TalosConfig   string
 }
 
-type talosConfig struct {
-	Context  string
-	Contexts map[string]*talosConfigContext
-}
-
-type talosConfigContext struct {
-	Target string
-	CA     string
-	Crt    string
-	Key    string
-}
-
 func (r *TalosConfigReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, options controller.Options) error {
 	r.Scheme = mgr.GetScheme()
 
@@ -148,6 +138,7 @@ func (r *TalosConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 	if err != nil {
 		return ctrl.Result{}, err
 	}
+
 	// Always attempt to Patch the TalosConfig object and status after each reconciliation.
 	defer func() {
 		// always update the readyCondition; the summary is represented using the "1 of x completed" notation.
@@ -156,14 +147,16 @@ func (r *TalosConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 				bootstrapv1alpha3.DataSecretAvailableCondition,
 			),
 		)
-		// Patch ObservedGeneration only if the reconciliation completed successfully
+
 		patchOpts := []patch.Option{
 			patch.WithOwnedConditions{
 				Conditions: []capiv1.ConditionType{
 					bootstrapv1alpha3.DataSecretAvailableCondition,
 				},
 			},
 		}
+
+		// Patch ObservedGeneration only if the reconciliation completed successfully
 		if rerr == nil {
 			patchOpts = append(patchOpts, patch.WithStatusObservedGeneration{})
 		}
@@ -214,11 +207,32 @@ func (r *TalosConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 		return ctrl.Result{}, err
 	}
 
+	if annotations.IsPaused(cluster, config) {
+		log.Info("Reconciliation is paused for this object")
+		return ctrl.Result{}, nil
+	}
+
+	tcScope := &TalosConfigScope{
+		Config:      config,
+		ConfigOwner: owner,
+		Cluster:     cluster,
+	}
+
 	// bail super early if it's already ready
 	if config.Status.Ready {
 		log.Info("ignoring an already ready config")
 		conditions.MarkTrue(config, bootstrapv1alpha3.DataSecretAvailableCondition)
-		return ctrl.Result{}, nil
+
+		// reconcile cluster-wide talosconfig
+		err = r.reconcileClientConfig(ctx, log, tcScope)
+
+		if err == nil {
+			conditions.MarkTrue(config, bootstrapv1alpha3.ClientConfigAvailableCondition)
+		} else {
+			conditions.MarkFalse(config, bootstrapv1alpha3.ClientConfigAvailableCondition, bootstrapv1alpha3.ClientConfigGenerationFailedReason, capiv1.ConditionSeverityError, "talosconfig generation failure: %s", err)
+		}
+
+		return ctrl.Result{}, err
 	}
 
 	// Wait patiently for the infrastructure to be ready
@@ -239,12 +253,6 @@ func (r *TalosConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 		return ctrl.Result{}, nil
 	}
 
-	tcScope := &TalosConfigScope{
-		Config:      config,
-		ConfigOwner: owner,
-		Cluster:     cluster,
-	}
-
 	if err = r.reconcileGenerate(ctx, tcScope); err != nil {
 		conditions.MarkFalse(config, bootstrapv1alpha3.DataSecretAvailableCondition, bootstrapv1alpha3.DataSecretGenerationFailedReason, capiv1.ConditionSeverityError, err.Error())
 
@@ -341,7 +349,7 @@ func (r *TalosConfigReconciler) reconcileGenerate(ctx context.Context, tcScope *
 	}
 
 	config.Status.DataSecretName = &dataSecretName
-	config.Status.TalosConfig = retData.TalosConfig
+	config.Status.TalosConfig = retData.TalosConfig //nolint:staticcheck // deprecated, for backwards compatibility only
 
 	return nil
 }
@@ -352,19 +360,31 @@ func (r *TalosConfigReconciler) reconcileDelete(ctx context.Context, config *boo
 	return ctrl.Result{}, nil
 }
 
-func genTalosConfigFile(clusterName string, certs *generate.Certs) (string, error) {
-	talosConfig := &talosConfig{
-		Context: clusterName,
-		Contexts: map[string]*talosConfigContext{
-			clusterName: {
-				Target: "",
-				CA:     base64.StdEncoding.EncodeToString(certs.OS.Crt),
-				Crt:    base64.StdEncoding.EncodeToString(certs.Admin.Crt),
-				Key:    base64.StdEncoding.EncodeToString(certs.Admin.Key),
-			},
+func genTalosConfigFile(clusterName string, bundle *generate.SecretsBundle, endpoints []string) (string, error) {
+	in := &generate.Input{
+		ClusterName: clusterName,
+		Certs: &generate.Certs{
+			OS: bundle.Certs.OS,
 		},
 	}
 
+	var err error
+
+	in.Certs.Admin, err = generate.NewAdminCertificateAndKey(
+		bundle.Clock.Now(),
+		bundle.Certs.OS,
+		role.MakeSet(role.Admin),
+		87600*time.Hour,
+	)
+	if err != nil {
+		return "", err
+	}
+
+	talosConfig, err := generate.Talosconfig(in, generate.WithEndpointList(endpoints))
+	if err != nil {
+		return "", err
+	}
+
 	talosConfigBytes, err := yaml.Marshal(talosConfig)
 	if err != nil {
 		return "", err
@@ -398,6 +418,15 @@ func (r *TalosConfigReconciler) userConfigs(ctx context.Context, scope *TalosCon
 
 	retBundle.BootstrapData = userConfigStr
 
+	if userConfig.Machine().Security().CA() != nil && len(userConfig.Machine().Security().CA().Crt) > 0 && len(userConfig.Machine().Security().CA().Key) > 0 {
+		bundle := generate.NewSecretsBundleFromConfig(generate.NewClock(), userConfig)
+
+		retBundle.TalosConfig, err = genTalosConfigFile(userConfig.Cluster().Name(), bundle, nil)
+		if err != nil {
+			r.Log.Error(err, "failed generating talosconfig for user-supplied machine configuration")
+		}
+	}
+
 	return retBundle, nil
 }
 
@@ -453,7 +482,7 @@ func (r *TalosConfigReconciler) genConfigs(ctx context.Context, scope *TalosConf
 
 	genOptions = append(genOptions, generate.WithVersionContract(versionContract))
 
-	secretBundle, err := r.getSecretsBundle(ctx, scope, scope.Cluster.Name+"-talos", genOptions...)
+	secretBundle, err := r.getSecretsBundle(ctx, scope, true, genOptions...)
 	if err != nil {
 		return retBundle, err
 	}
@@ -476,7 +505,7 @@ func (r *TalosConfigReconciler) genConfigs(ctx context.Context, scope *TalosConf
 		return retBundle, err
 	}
 
-	tcString, err := genTalosConfigFile(input.ClusterName, input.Certs)
+	tcString, err := genTalosConfigFile(input.ClusterName, secretBundle, nil)
 	if err != nil {
 		return retBundle, err
 	}

go.mod

@@ -12,6 +12,7 @@ require (
 	github.com/talos-systems/talos/pkg/machinery v0.13.0
 	golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6
 	gopkg.in/yaml.v2 v2.4.0
+	inet.af/netaddr v0.0.0-20210903134321-85fa6c94624e
 	k8s.io/api v0.21.4
 	k8s.io/apiextensions-apiserver v0.21.4
 	k8s.io/apimachinery v0.21.4
@@ -28,6 +29,8 @@ require (
 	github.com/cespare/xxhash/v2 v2.1.1 // indirect
 	github.com/containerd/go-cni v1.1.0 // indirect
 	github.com/containernetworking/cni v1.0.1 // indirect
+	github.com/coredns/caddy v1.1.0 // indirect
+	github.com/coredns/corefile-migration v1.0.13 // indirect
 	github.com/cosi-project/runtime v0.0.0-20210906201716-5cb7f5002d77 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/docker/distribution v2.7.1+incompatible // indirect
@@ -85,6 +88,8 @@ require (
 	go.uber.org/atomic v1.7.0 // indirect
 	go.uber.org/multierr v1.7.0 // indirect
 	go.uber.org/zap v1.19.0 // indirect
+	go4.org/intern v0.0.0-20210108033219-3eb7198706b2 // indirect
+	go4.org/unsafe/assume-no-moving-gc v0.0.0-20201222180813-1025295fd063 // indirect
 	golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 // indirect
 	golang.org/x/net v0.0.0-20210525063256-abc453219eb5 // indirect
 	golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914 // indirect