Skip to content

Commit 755a2dd

Browse files
committed
fix: update Talos machinery to 0.12, fix secrets persistence
There are several important changes: * `talosVersion` now defaults to whatever current Talos version is vs. `0.8` as it was before; we don't expect 0.8 being used to bootstrap new clusters * Talos machinery bumped to latest 0.12 * secrets for the configuration are now preserved directly as machinery's `SecretsBundle` providing forward compatibility with future machine configuration changes: fixes persistence for cluster.id/secret for 0.12 * migration path from "legacy" secret format to new one * general clean ups, less dependency on "init" node type, provider should work fine in init-less flow. Signed-off-by: Andrey Smirnov <[email protected]>
1 parent f91b032 commit 755a2dd

File tree

6 files changed

+221
-174
lines changed

6 files changed

+221
-174
lines changed

controllers/secrets.go

Lines changed: 85 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -34,27 +34,74 @@ func (r *TalosConfigReconciler) fetchSecret(ctx context.Context, config *bootstr
3434
return retSecret, nil
3535
}
3636

37-
func (r *TalosConfigReconciler) writeInputSecret(ctx context.Context, scope *TalosConfigScope, input *generate.Input) (*corev1.Secret, error) {
37+
// getSecretsBundle either generates or loads existing secret.
38+
func (r *TalosConfigReconciler) getSecretsBundle(ctx context.Context, scope *TalosConfigScope, secretName string, opts ...generate.GenOption) (*generate.SecretsBundle, error) {
39+
var secretsBundle *generate.SecretsBundle
3840

39-
certMarshal, err := yaml.Marshal(input.Certs)
40-
if err != nil {
41-
return nil, err
42-
}
41+
retry:
42+
secret, err := r.fetchSecret(ctx, scope.Config, secretName)
4343

44-
kubeSecretsMarshal, err := yaml.Marshal(input.Secrets)
45-
if err != nil {
46-
return nil, err
44+
switch {
45+
case err != nil && k8serrors.IsNotFound(err):
46+
// no cluster secret yet, generate new one
47+
secretsBundle, err = generate.NewSecretsBundle(generate.NewClock(), opts...)
48+
if err != nil {
49+
return nil, fmt.Errorf("error generating new secrets bundle: %w", err)
50+
}
51+
52+
if err = r.writeSecretsBundleSecret(ctx, scope, secretName, secretsBundle); err != nil {
53+
if k8serrors.IsAlreadyExists(err) {
54+
// conflict on creation, retry loading
55+
goto retry
56+
}
57+
58+
return nil, fmt.Errorf("error writing secrets bundle: %w", err)
59+
}
60+
case err != nil:
61+
return nil, fmt.Errorf("error reading secrets bundle: %w", err)
62+
default:
63+
// successfully loaded secret, initialize secretsBundle from it
64+
secretsBundle = &generate.SecretsBundle{
65+
Clock: generate.NewClock(),
66+
}
67+
68+
if _, ok := secret.Data["bundle"]; ok {
69+
// new format
70+
if err = yaml.Unmarshal(secret.Data["bundle"], secretsBundle); err != nil {
71+
return nil, fmt.Errorf("error unmarshaling secrets bundle: %w", err)
72+
}
73+
} else {
74+
// legacy format
75+
if err = yaml.Unmarshal(secret.Data["certs"], &secretsBundle.Certs); err != nil {
76+
return nil, fmt.Errorf("error unmarshaling certs: %w", err)
77+
}
78+
79+
if err = yaml.Unmarshal(secret.Data["kubeSecrets"], &secretsBundle.Secrets); err != nil {
80+
return nil, fmt.Errorf("error unmarshaling secrets: %w", err)
81+
}
82+
83+
if err = yaml.Unmarshal(secret.Data["trustdInfo"], &secretsBundle.TrustdInfo); err != nil {
84+
return nil, fmt.Errorf("error unmarshaling trustd info: %w", err)
85+
}
86+
87+
// not stored in legacy format, use empty values
88+
secretsBundle.Cluster = &generate.Cluster{}
89+
}
4790
}
4891

49-
trustdInfoMarshal, err := yaml.Marshal(input.TrustdInfo)
92+
return secretsBundle, nil
93+
}
94+
95+
func (r *TalosConfigReconciler) writeSecretsBundleSecret(ctx context.Context, scope *TalosConfigScope, secretName string, secretsBundle *generate.SecretsBundle) error {
96+
bundle, err := yaml.Marshal(secretsBundle)
5097
if err != nil {
51-
return nil, err
98+
return fmt.Errorf("error marshaling secrets bundle: %w", err)
5299
}
53100

54-
certSecret := &corev1.Secret{
101+
secret := &corev1.Secret{
55102
ObjectMeta: metav1.ObjectMeta{
56103
Namespace: scope.Config.Namespace,
57-
Name: scope.Cluster.Name + "-talos",
104+
Name: secretName,
58105
Labels: map[string]string{
59106
capiv1.ClusterLabelName: scope.Cluster.Name,
60107
},
@@ -63,45 +110,43 @@ func (r *TalosConfigReconciler) writeInputSecret(ctx context.Context, scope *Tal
63110
},
64111
},
65112
Data: map[string][]byte{
66-
"certs": certMarshal,
67-
"kubeSecrets": kubeSecretsMarshal,
68-
"trustdInfo": trustdInfoMarshal,
113+
"bundle": bundle,
69114
},
70115
}
71116

72-
err = r.Client.Create(ctx, certSecret)
73-
if err != nil {
74-
return nil, err
75-
}
76-
return certSecret, nil
117+
return r.Client.Create(ctx, secret)
77118
}
78119

79120
func (r *TalosConfigReconciler) writeK8sCASecret(ctx context.Context, scope *TalosConfigScope, certs *x509.PEMEncodedCertificateAndKey) error {
80121
// Create ca secret only if it doesn't already exist
81122
_, err := r.fetchSecret(ctx, scope.Config, scope.Cluster.Name+"-ca")
82-
if k8serrors.IsNotFound(err) {
83-
certSecret := &corev1.Secret{
84-
ObjectMeta: metav1.ObjectMeta{
85-
Namespace: scope.Config.Namespace,
86-
Name: scope.Cluster.Name + "-ca",
87-
Labels: map[string]string{
88-
capiv1.ClusterLabelName: scope.Cluster.Name,
89-
},
90-
OwnerReferences: []metav1.OwnerReference{
91-
*metav1.NewControllerRef(scope.Cluster, capiv1.GroupVersion.WithKind("Cluster")),
92-
},
123+
if err == nil {
124+
return nil
125+
}
126+
127+
if !k8serrors.IsNotFound(err) {
128+
return err
129+
}
130+
131+
certSecret := &corev1.Secret{
132+
ObjectMeta: metav1.ObjectMeta{
133+
Namespace: scope.Config.Namespace,
134+
Name: scope.Cluster.Name + "-ca",
135+
Labels: map[string]string{
136+
capiv1.ClusterLabelName: scope.Cluster.Name,
93137
},
94-
Data: map[string][]byte{
95-
"tls.crt": certs.Crt,
96-
"tls.key": certs.Key,
138+
OwnerReferences: []metav1.OwnerReference{
139+
*metav1.NewControllerRef(scope.Cluster, capiv1.GroupVersion.WithKind("Cluster")),
97140
},
98-
}
141+
},
142+
Data: map[string][]byte{
143+
"tls.crt": certs.Crt,
144+
"tls.key": certs.Key,
145+
},
146+
}
99147

100-
err = r.Client.Create(ctx, certSecret)
101-
if err != nil {
102-
return err
103-
}
104-
} else if err != nil {
148+
err = r.Client.Create(ctx, certSecret)
149+
if err != nil && !k8serrors.IsAlreadyExists(err) {
105150
return err
106151
}
107152

controllers/talosconfig_controller.go

Lines changed: 16 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -19,11 +19,10 @@ import (
1919
"github.com/talos-systems/talos/pkg/machinery/config/configpatcher"
2020
"github.com/talos-systems/talos/pkg/machinery/config/types/v1alpha1"
2121
"github.com/talos-systems/talos/pkg/machinery/config/types/v1alpha1/generate"
22-
configmachine "github.com/talos-systems/talos/pkg/machinery/config/types/v1alpha1/machine"
22+
"github.com/talos-systems/talos/pkg/machinery/config/types/v1alpha1/machine"
2323
"github.com/talos-systems/talos/pkg/machinery/constants"
2424
"gopkg.in/yaml.v2"
2525
apierrors "k8s.io/apimachinery/pkg/api/errors"
26-
k8serrors "k8s.io/apimachinery/pkg/api/errors"
2726
"k8s.io/apimachinery/pkg/runtime"
2827
capiv1 "sigs.k8s.io/cluster-api/api/v1alpha3"
2928
bsutil "sigs.k8s.io/cluster-api/bootstrap/util"
@@ -48,7 +47,7 @@ const (
4847
)
4948

5049
var (
51-
defaultVersionContract = config.TalosVersion0_8
50+
defaultVersionContract = config.TalosVersionCurrent
5251
)
5352

5453
// TalosConfigReconciler reconciles a TalosConfig object
@@ -206,9 +205,11 @@ func (r *TalosConfigReconciler) Reconcile(req ctrl.Request) (_ ctrl.Result, rerr
206205

207206
var retData *TalosConfigBundle
208207

209-
switch config.Spec.GenerateType {
208+
machineType, _ := machine.ParseType(config.Spec.GenerateType) //nolint:errcheck // handle errors later
209+
210+
switch {
210211
// Slurp and use user-supplied configs
211-
case "none":
212+
case config.Spec.GenerateType == "none":
212213
if config.Spec.Data == "" {
213214
return ctrl.Result{}, errors.New("failed to specify config data with none generate type")
214215
}
@@ -218,14 +219,14 @@ func (r *TalosConfigReconciler) Reconcile(req ctrl.Request) (_ ctrl.Result, rerr
218219
}
219220

220221
// Generate configs on the fly
221-
case "init", "controlplane", "join":
222+
case machineType != machine.TypeUnknown:
222223
retData, err = r.genConfigs(ctx, tcScope)
223224
if err != nil {
224225
return ctrl.Result{}, err
225226
}
226227

227228
default:
228-
return ctrl.Result{}, errors.New("unknown generate type specified")
229+
return ctrl.Result{}, fmt.Errorf("unknown generate type specified: %q", config.Spec.GenerateType)
229230
}
230231

231232
// Handle patches to the machine config if they were specified
@@ -319,11 +320,8 @@ func (r *TalosConfigReconciler) userConfigs(ctx context.Context, scope *TalosCon
319320
}
320321

321322
// Create the secret with kubernetes certs so a kubeconfig can be generated
322-
if userConfig.Machine().Type() == configmachine.TypeInit {
323-
err = r.writeK8sCASecret(ctx, scope, userConfig.Cluster().CA())
324-
if err != nil {
325-
return retBundle, err
326-
}
323+
if err = r.writeK8sCASecret(ctx, scope, userConfig.Cluster().CA()); err != nil {
324+
return retBundle, err
327325
}
328326

329327
userConfigStr, err := userConfig.String()
@@ -341,12 +339,9 @@ func (r *TalosConfigReconciler) genConfigs(ctx context.Context, scope *TalosConf
341339
retBundle := &TalosConfigBundle{}
342340

343341
// Determine what type of node this is
344-
machineType := configmachine.TypeJoin
345-
switch scope.Config.Spec.GenerateType {
346-
case "init":
347-
machineType = configmachine.TypeInit
348-
case "controlplane":
349-
machineType = configmachine.TypeControlPlane
342+
machineType, err := machine.ParseType(scope.Config.Spec.GenerateType)
343+
if err != nil {
344+
machineType = machine.TypeWorker
350345
}
351346

352347
// Allow user to override default kube version.
@@ -391,12 +386,13 @@ func (r *TalosConfigReconciler) genConfigs(ctx context.Context, scope *TalosConf
391386

392387
genOptions = append(genOptions, generate.WithVersionContract(versionContract))
393388

394-
secretBundle, err := generate.NewSecretsBundle(generate.NewClock(), genOptions...)
389+
secretBundle, err := r.getSecretsBundle(ctx, scope, scope.Cluster.Name+"-talos", genOptions...)
395390
if err != nil {
396391
return retBundle, err
397392
}
398393

399394
APIEndpointPort := strconv.Itoa(int(scope.Cluster.Spec.ControlPlaneEndpoint.Port))
395+
400396
input, err := generate.NewInput(
401397
scope.Cluster.Name,
402398
"https://"+scope.Cluster.Spec.ControlPlaneEndpoint.Host+":"+APIEndpointPort,
@@ -408,51 +404,11 @@ func (r *TalosConfigReconciler) genConfigs(ctx context.Context, scope *TalosConf
408404
return retBundle, err
409405
}
410406

411-
// Stash our generated input secrets so that we can reuse them for other nodes
412-
inputSecret, err := r.fetchSecret(ctx, scope.Config, scope.Cluster.Name+"-talos")
413-
if machineType == configmachine.TypeInit && k8serrors.IsNotFound(err) {
414-
inputSecret, err = r.writeInputSecret(ctx, scope, input)
415-
if err != nil {
416-
return retBundle, err
417-
}
418-
} else if err != nil {
419-
return retBundle, err
420-
}
421-
422407
// Create the secret with kubernetes certs so a kubeconfig can be generated
423-
_, err = r.fetchSecret(ctx, scope.Config, scope.Cluster.Name+"-ca")
424-
if machineType == configmachine.TypeInit && k8serrors.IsNotFound(err) {
425-
err = r.writeK8sCASecret(ctx, scope, input.Certs.K8s)
426-
if err != nil {
427-
return retBundle, err
428-
}
429-
} else if err != nil {
430-
return retBundle, err
431-
}
432-
433-
certs := &generate.Certs{}
434-
kubeSecrets := &generate.Secrets{}
435-
trustdInfo := &generate.TrustdInfo{}
436-
437-
err = yaml.Unmarshal(inputSecret.Data["certs"], certs)
438-
if err != nil {
408+
if err = r.writeK8sCASecret(ctx, scope, input.Certs.K8s); err != nil {
439409
return retBundle, err
440410
}
441411

442-
err = yaml.Unmarshal(inputSecret.Data["kubeSecrets"], kubeSecrets)
443-
if err != nil {
444-
return retBundle, err
445-
}
446-
447-
err = yaml.Unmarshal(inputSecret.Data["trustdInfo"], trustdInfo)
448-
if err != nil {
449-
return retBundle, err
450-
}
451-
452-
input.Certs = certs
453-
input.Secrets = kubeSecrets
454-
input.TrustdInfo = trustdInfo
455-
456412
tcString, err := genTalosConfigFile(input.ClusterName, input.Certs)
457413
if err != nil {
458414
return retBundle, err

go.mod

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -8,15 +8,14 @@ require (
88
github.com/go-logr/logr v0.1.0
99
github.com/spf13/pflag v1.0.5
1010
github.com/stretchr/testify v1.7.0
11-
github.com/talos-systems/crypto v0.3.1
12-
github.com/talos-systems/talos/pkg/machinery v0.11.3
13-
golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71
11+
github.com/talos-systems/crypto v0.3.2
12+
github.com/talos-systems/talos/pkg/machinery v0.12.3-0.20210920195258-7e63e43eb399
13+
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c
1414
gopkg.in/yaml.v2 v2.4.0
1515
k8s.io/api v0.17.9
1616
k8s.io/apiextensions-apiserver v0.17.9
1717
k8s.io/apimachinery v0.17.9
1818
k8s.io/client-go v0.17.9
19-
k8s.io/utils v0.0.0-20200619165400-6e3d28b6ed19
2019
sigs.k8s.io/cluster-api v0.3.22
2120
sigs.k8s.io/controller-runtime v0.5.14
2221
)
@@ -28,7 +27,7 @@ require (
2827
github.com/cespare/xxhash/v2 v2.1.1 // indirect
2928
github.com/containerd/go-cni v1.0.2 // indirect
3029
github.com/containernetworking/cni v0.8.1 // indirect
31-
github.com/cosi-project/runtime v0.0.0-20210625174835-93ead370bf57 // indirect
30+
github.com/cosi-project/runtime v0.0.0-20210707150857-25f235cd0682 // indirect
3231
github.com/davecgh/go-spew v1.1.1 // indirect
3332
github.com/docker/distribution v2.7.1+incompatible // indirect
3433
github.com/drone/envsubst v1.0.3-0.20200709223903-efdb65b94e5a // indirect
@@ -79,21 +78,21 @@ require (
7978
github.com/spf13/jwalterweatherman v1.0.0 // indirect
8079
github.com/spf13/viper v1.6.2 // indirect
8180
github.com/subosito/gotenv v1.2.0 // indirect
82-
github.com/talos-systems/go-blockdevice v0.2.1 // indirect
81+
github.com/talos-systems/go-blockdevice v0.2.3 // indirect
8382
github.com/talos-systems/net v0.3.0 // indirect
8483
go.uber.org/atomic v1.7.0 // indirect
85-
go.uber.org/multierr v1.6.0 // indirect
86-
go.uber.org/zap v1.16.0 // indirect
84+
go.uber.org/multierr v1.7.0 // indirect
85+
go.uber.org/zap v1.18.1 // indirect
8786
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 // indirect
8887
golang.org/x/net v0.0.0-20210525063256-abc453219eb5 // indirect
8988
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect
9089
golang.org/x/text v0.3.6 // indirect
9190
golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect
9291
gomodules.xyz/jsonpatch/v2 v2.0.1 // indirect
9392
google.golang.org/appengine v1.6.6 // indirect
94-
google.golang.org/genproto v0.0.0-20210617175327-b9e0b3197ced // indirect
95-
google.golang.org/grpc v1.38.0 // indirect
96-
google.golang.org/protobuf v1.26.0 // indirect
93+
google.golang.org/genproto v0.0.0-20210722135532-667f2b7c528f // indirect
94+
google.golang.org/grpc v1.40.0 // indirect
95+
google.golang.org/protobuf v1.27.1 // indirect
9796
gopkg.in/fsnotify.v1 v1.4.7 // indirect
9897
gopkg.in/inf.v0 v0.9.1 // indirect
9998
gopkg.in/ini.v1 v1.51.0 // indirect
@@ -104,5 +103,6 @@ require (
104103
k8s.io/klog v1.0.0 // indirect
105104
k8s.io/klog/v2 v2.0.0 // indirect
106105
k8s.io/kube-openapi v0.0.0-20200410145947-bcb3869e6f29 // indirect
106+
k8s.io/utils v0.0.0-20200619165400-6e3d28b6ed19 // indirect
107107
sigs.k8s.io/yaml v1.2.0 // indirect
108108
)

0 commit comments

Comments
 (0)