Switch from in-memory CA key/pair to in-cluster spec

Author: deitch

Gopkg.lock

@@ -40,7 +40,7 @@ To deploy a cluster:
    * `provider-components.yaml` - note that this file _will_ contain your secrets, specifically `PACKET_API_KEY`, to be loaded into the cluster
    * `addons.yaml` - note that this file _will_ contain your secrets, specifically `PACKET_API_KEY`, to be loaded into the cluster
 1. If desired, edit the following files:
-   * `cluster.yaml` - to change parameters or settings, including network CIDRs
+   * `cluster.yaml` - to change parameters or settings, including network CIDRs, and, if desired, your own CA certificate and key
    * `machines.yaml` - to change parameters or settings, including machine types and quantity
 1. Run `clusterctl` with the appropriate command.
 
@@ -68,6 +68,16 @@ Run `clusterctl create cluster --help` for more options, for example to use an e
 1. Deploy add-on components, e.g. the [packet cloud-controller-manager](https://github.com/packethost/packet-ccm) and the [packet cloud storage interface provider](https://github.com/packethost/csi-packet)
 1. If a new bootstrap cluster was created, terminate it
 
+#### Defaults
+
+If you do not change the generated `yaml` files, it will use defaults. You can look in the `*.yaml.template` files in [cmd/clusterctl/examples/packet/](./cmd/clusterctl/examples/packet/) for details.
+
+* CA key/certificate: leave blank, which will cause the `manager` to create one.
+* service CIDR: `172.25.0.0/16`
+* pod CIDR: `172.26.0.0/16`
+* service domain: `cluster.local`
+* cluster name: `test1-<random>`, where random is a random 5-character string containing the characters `a-z0-9`
+
 #### About Those Secrets
 
 Notice that the API key is load into _two_ separate files, `provider-components.yaml` and `addons.yaml`. This is unfortunately necessary.
@@ -132,19 +142,17 @@ The Packet cluster-api provider follows the standard design for cluster-api. It
 The actual machines are deployed using `kubeadm`. The deployment process uses the following process.
 
 1. When a new `Cluster` is created:
-   * create a new CA certificate and key, encrypt and save them in a bootstrap controller Kubernetes secret
-   * create a new kubeadm token, encrypt and save in a bootstrap controller Kubernetes secret. This 
-   * create an admin user certificate using the CA certificate and key, encrypt and save them in a bootstrap controller Kubernetes secret
+   * if the `ClusterSpec` does not include a CA key/certificate pair, create one and save it on the `Cluster` object
 2. When a new master `Machine` is created:
-   * retrieve the CA certificate and key from the bootstrap controller Kubernetes secret
-   * retrieve the kubeadm token from the bootstrap controller Kubernetes secret
+   * retrieve the CA certificate and key from the `Cluster` object
    * launch a new server instance on Packet
-   * set the `cloud-init` on the instance to run `kubeadm init`, passing it the CA certificate and key and kubeadm token
+   * set the `cloud-init` on the instance to run `kubeadm init`, passing it the CA certificate and key
 3. When a new worker `Machine` is created:
-   * retrieve the kubeadm token from the Kubernetes secret
+   * check if the cluster is ready, i.e. has a valid endpoint; if not, retry every 15 seconds
+   * generate a new kubeadm bootstrap token and save it to the workload cluster
    * launch a new server instance on Packet
-   * set the `cloud-init` on the instance to run `kubeadm join`, passing it the kubeadm token
-4. When a user requests the kubeconfig via `clusterctl`, it retrieves it from the Kubernetes secret and passes it to the user
+   * set the `cloud-init` on the instance to run `kubeadm join`, passing it the newly generated kubeadm token
+4. When a user requests the kubeconfig via `clusterctl`, generate a new one using the CA key/certificate pair
 
 
 ## Building
@@ -218,7 +226,7 @@ You can run the `manager` locally on your own laptop in order to ease and speed
 1. Set your `KUBECONFIG` to point to that cluster, e.g. `export KUBECONFIG=...`
 1. Create a local OS/arch `manager` binary, essentially `make manager`. This will save it as `bin/manager-<os>-<arch>`, e.g. `bin/manager-linux-arm64` or `bin/manager-darwin-amd64`
 1. Generate your yaml `./generate-yaml.sh`
-1. Run the manager against the cluster with the local configs, `bin/manager-darwin-amd64 -config ./config/default/machine_configs.yaml -ca-cache ./out/cache.json`
+1. Run the manager against the cluster with the local configs, `bin/manager-darwin-amd64 -config ./config/default/machine_configs.yaml`
 
 In the above example:
 

README.md

@@ -14,3 +14,6 @@ spec:
         apiVersion: "packetprovider/v1alpha1"
         kind: "PacketClusterProviderSpec"
         projectID: "$PROJECT_ID"
+        caKeyPair:
+          cert: ""
+          key: ""

cmd/clusterctl/examples/packet/cluster.yaml.template

@@ -53,7 +53,6 @@ func main() {
 	metricsAddr := flag.String("metrics-addr", ":8080", "The address the metric endpoint binds to.")
 
 	machineSetupConfig := flag.String("config", "/etc/machineconfig/machine_configs.yaml", "path to the machine setup config")
-	caCache := flag.String("ca-cache", "", "path to file to save cluster credentials; should be used for development purposes only, as it saves actual CA credentials")
 	flag.Parse()
 
 	log := logf.Log.WithName("packet-controller-manager")
@@ -79,9 +78,8 @@ func main() {
 	}
 	// get a deployer, which is needed at various stages
 	deployer, err := deployer.New(deployer.Params{
-		Client:  client,
-		Port:    controlPort,
-		CACache: *caCache,
+		Client: client,
+		Port:   controlPort,
 	})
 	if err != nil {
 		klog.Fatalf(err.Error())

cmd/manager/main.go

@@ -19,6 +19,20 @@ spec:
             of an object. Servers should convert recognized schemas to the latest
             internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources'
           type: string
+        caKeyPair:
+          description: CAKeyPair is the key pair for ca certs.
+          properties:
+            cert:
+              description: base64 encoded cert and key
+              format: byte
+              type: string
+            key:
+              format: byte
+              type: string
+          required:
+          - cert
+          - key
+          type: object
         kind:
           description: 'Kind is a string value representing the REST resource this
             object represents. Servers may infer this from the endpoint the client

config/crds/packetprovider_v1alpha1_packetclusterproviderspec.yaml

@@ -33,6 +33,16 @@ type PacketClusterProviderSpec struct {
 	metav1.ObjectMeta `json:"metadata,omitempty"`
 
 	ProjectID string `json:"projectID"`
+
+	// CAKeyPair is the key pair for ca certs.
+	CAKeyPair KeyPair `json:"caKeyPair,omitempty"`
+}
+
+// KeyPair is how operators can supply custom keypairs for kubeadm to use.
+type KeyPair struct {
+	// base64 encoded cert and key
+	Cert []byte `json:"cert"`
+	Key  []byte `json:"key"`
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

pkg/apis/packetprovider/v1alpha1/packetclusterproviderspec_types.go

@@ -34,7 +34,12 @@ func TestStoragePacketClusterProviderSpec(t *testing.T) {
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "foo",
 			Namespace: "default",
-		}}
+		},
+		CAKeyPair: KeyPair{
+			Key:  []byte{10, 20, 30},
+			Cert: []byte{10, 20, 30},
+		},
+	}
 	g := gomega.NewGomegaWithT(t)
 
 	// Test Create

pkg/apis/packetprovider/v1alpha1/packetclusterproviderspec_types_test.go

@@ -24,6 +24,7 @@ import (
 
 	"github.com/packethost/cluster-api-provider-packet/pkg/cloud/packet/ca"
 	"github.com/packethost/cluster-api-provider-packet/pkg/cloud/packet/deployer"
+	"github.com/packethost/cluster-api-provider-packet/pkg/cloud/packet/util"
 	clusterv1 "sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1"
 	client "sigs.k8s.io/cluster-api/pkg/client/clientset_generated/clientset/typed/cluster/v1alpha1"
 	controllerError "sigs.k8s.io/cluster-api/pkg/controller/error"
@@ -60,16 +61,39 @@ func (a *Actuator) Reconcile(cluster *clusterv1.Cluster) error {
 	log.Printf("Reconciling cluster %v.", cluster.Name)
 	// save the original status
 	clusterCopy := cluster.DeepCopy()
+	// get a client we can use
+	var (
+		clusterClient  client.ClusterInterface
+		updatedCluster *clusterv1.Cluster
+	)
+	if a.clustersGetter != nil {
+		clusterClient = a.clustersGetter.Clusters(cluster.Namespace)
+	}
 	// ensure that we have a CA cert/key and save it
-	if cert, _ := a.deployer.GetCA(cluster.Name); cert == nil {
+	c, err := util.ClusterProviderFromProviderConfig(cluster.Spec.ProviderSpec)
+	if err != nil {
+		return fmt.Errorf("unable to unpack cluster provider for cluster %s: %v", cluster.Name, err)
+	}
+	if len(c.CAKeyPair.Cert) == 0 || len(c.CAKeyPair.Cert) == 0 {
 		caCertAndKey, err := ca.GenerateCACertAndKey(cluster.Name, "")
 		if err != nil {
-			return fmt.Errorf("unable to generate CA cert and key: %v", err)
+			return fmt.Errorf("unable to generate CA cert and key for cluster %s: %v", cluster.Name, err)
 		}
-		err = a.deployer.PutCA(cluster.Name, caCertAndKey)
+		c.CAKeyPair.Cert = caCertAndKey.Certificate
+		c.CAKeyPair.Key = caCertAndKey.PrivateKey
+		// update cluster spec
+		spec, err := util.ClusterProviderConfigFromProvider(c)
 		if err != nil {
-			return fmt.Errorf("unable to save CA cert and key: %v", err)
+			return fmt.Errorf("unable to convert newly generated provider spec with CA key/certificate to provider config for %s: %v", cluster.Name, err)
+		}
+		cluster.Spec.ProviderSpec = spec
+		log.Printf("saving updated cluster spec %s", cluster.Name)
+		if updatedCluster, err = clusterClient.Update(cluster); err != nil {
+			msg := fmt.Sprintf("failed to save updated cluster %s: %v", cluster.Name, err)
+			log.Printf(msg)
+			return fmt.Errorf(msg)
 		}
+		cluster = updatedCluster
 	}
 	// ensure that we save the correct IP address for the cluster
 	address, err := a.deployer.GetIP(cluster, nil)
@@ -91,10 +115,6 @@ func (a *Actuator) Reconcile(cluster *clusterv1.Cluster) error {
 		}
 	}
 
-	var clusterClient client.ClusterInterface
-	if a.clustersGetter != nil {
-		clusterClient = a.clustersGetter.Clusters(cluster.Namespace)
-	}
 	if !reflect.DeepEqual(cluster.Status, clusterCopy.Status) {
 		log.Printf("saving updated cluster status %s", cluster.Name)
 		if _, err := clusterClient.UpdateStatus(cluster); err != nil {
@@ -110,7 +130,5 @@ func (a *Actuator) Reconcile(cluster *clusterv1.Cluster) error {
 // Delete deletes a cluster and is invoked by the Cluster Controller
 func (a *Actuator) Delete(cluster *clusterv1.Cluster) error {
 	log.Printf("Deleting cluster %v.", cluster.Name)
-	// remove the CA cert key
-	a.deployer.DeleteCA(cluster.Name)
 	return nil
 }

pkg/apis/packetprovider/v1alpha1/zz_generated.deepcopy.go

@@ -28,7 +28,6 @@ import (
 	"github.com/packethost/cluster-api-provider-packet/pkg/cloud/packet/util"
 	"github.com/packethost/packngo"
 	clusterv1 "sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1"
-	"sigs.k8s.io/cluster-api/pkg/cert"
 )
 
 const (
@@ -78,6 +77,10 @@ func (a *Actuator) Create(ctx context.Context, cluster *clusterv1.Cluster, machi
 	if err != nil {
 		return fmt.Errorf("Unable to read providerSpec from machine config: %v", err)
 	}
+	clusterConfig, err := util.ClusterProviderFromProviderConfig(cluster.Spec.ProviderSpec)
+	if err != nil {
+		return fmt.Errorf("unable to unpack cluster provider: %v", err)
+	}
 
 	tags := []string{
 		util.GenerateMachineTag(string(machine.UID)),
@@ -97,16 +100,8 @@ func (a *Actuator) Create(ctx context.Context, cluster *clusterv1.Cluster, machi
 	)
 	if machine.Spec.Versions.ControlPlane != "" {
 		role = "master"
-		// generate a cluster CA cert and key
-		var (
-			caCertAndKey *cert.CertificateAuthority
-			ok           bool
-		)
-		if caCertAndKey, ok = a.deployer.Certs[cluster.Name]; !ok {
-			return fmt.Errorf("Unable to read CA cert/key for uninitialized cluster %s", cluster.Name)
-		}
-		caCert = caCertAndKey.Certificate
-		caKey = caCertAndKey.PrivateKey
+		caCert = clusterConfig.CAKeyPair.Cert
+		caKey = clusterConfig.CAKeyPair.Key
 		tags = append(tags, util.MasterTag)
 	} else {
 		token, err = a.deployer.NewBootstrapToken(cluster)