pkg/infrastructure: add capi provider

Implements the infrastructure provider interface with the CAPI system.
This encapsulates the CAPI implementation similar to Terraform. It also
maintains pkg/infrastructure/platform.go (and build variants) as the
canonical source of truth for choosing an infrastructure provider.

This also adds an interface that cloud platforms utilizing the CAPI
provisioning should implement to provision additional resources.

Author: patrickdillon

pkg/asset/cluster/cluster.go

@@ -5,31 +5,23 @@ import (
 	"fmt"
 	"path/filepath"
 	"strings"
-	"time"
 
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
-	"gopkg.in/yaml.v2"
-	apierrors "k8s.io/apimachinery/pkg/api/errors"
-	"k8s.io/apimachinery/pkg/util/wait"
-	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
-	utilkubeconfig "sigs.k8s.io/cluster-api/util/kubeconfig"
-	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/manager/signals"
 
 	"github.com/openshift/installer/pkg/asset"
 	"github.com/openshift/installer/pkg/asset/cluster/aws"
 	"github.com/openshift/installer/pkg/asset/cluster/azure"
 	"github.com/openshift/installer/pkg/asset/cluster/openstack"
 	"github.com/openshift/installer/pkg/asset/cluster/tfvars"
+	"github.com/openshift/installer/pkg/asset/ignition/bootstrap"
+	"github.com/openshift/installer/pkg/asset/ignition/machine"
 	"github.com/openshift/installer/pkg/asset/installconfig"
-	awsconfig "github.com/openshift/installer/pkg/asset/installconfig/aws"
 	"github.com/openshift/installer/pkg/asset/kubeconfig"
-	"github.com/openshift/installer/pkg/asset/manifests/capiutils"
+	"github.com/openshift/installer/pkg/asset/machines"
 	capimanifests "github.com/openshift/installer/pkg/asset/manifests/clusterapi"
 	"github.com/openshift/installer/pkg/asset/password"
 	"github.com/openshift/installer/pkg/asset/quota"
-	"github.com/openshift/installer/pkg/clusterapi"
 	infra "github.com/openshift/installer/pkg/infrastructure/platform"
 	typesaws "github.com/openshift/installer/pkg/types/aws"
 	typesazure "github.com/openshift/installer/pkg/types/azure"
@@ -72,6 +64,9 @@ func (c *Cluster) Dependencies() []asset.Asset {
 		&password.KubeadminPassword{},
 		&capimanifests.Cluster{},
 		&kubeconfig.AdminClient{},
+		&bootstrap.Bootstrap{},
+		&machine.Master{},
+		&machines.ClusterAPI{},
 	}
 }
 
@@ -98,31 +93,14 @@ func (c *Cluster) Generate(parents asset.Parents) (err error) {
 		return errors.New("cluster cannot be created with bootstrapInPlace set")
 	}
 
-	// Check if we're using Cluster API.
-	if capiutils.IsEnabled(installConfig) {
-		// TODO(vincepri): The context should be passed down from the caller,
-		// although today the Asset interface doesn't allow it, refactor once it does.
-		ctx, cancel := context.WithCancel(signals.SetupSignalHandler())
-		go func() {
-			<-ctx.Done()
-			cancel()
-			clusterapi.System().Teardown()
-		}()
-
-		return c.provisionWithClusterAPI(ctx, parents, installConfig, clusterID)
-	}
-
-	// Otherwise, use the normal path.
-	return c.provision(installConfig, clusterID, terraformVariables, parents)
-}
-
-func (c *Cluster) provision(installConfig *installconfig.InstallConfig, clusterID *installconfig.ClusterID, terraformVariables *tfvars.TerraformVariables, parents asset.Parents) error {
 	platform := installConfig.Config.Platform.Name()
 
 	if azure := installConfig.Config.Platform.Azure; azure != nil && azure.CloudName == typesazure.StackCloud {
 		platform = typesazure.StackTerraformName
 	}
 
+	// TODO(padillon): determine whether CAPI handles tagging shared subnets, in which case we should be able
+	// to encapsulate these into the terraform package.
 	logrus.Infof("Creating infrastructure resources...")
 	switch platform {
 	case typesaws.Name:
@@ -154,137 +132,6 @@ func (c *Cluster) provision(installConfig *installconfig.InstallConfig, clusterI
 	return nil
 }
 
-func (c *Cluster) provisionWithClusterAPI(ctx context.Context, parents asset.Parents, installConfig *installconfig.InstallConfig, clusterID *installconfig.ClusterID) error {
-	capiManifests := &capimanifests.Cluster{}
-	clusterKubeconfigAsset := &kubeconfig.AdminClient{}
-	parents.Get(
-		capiManifests,
-		clusterKubeconfigAsset,
-	)
-
-	// Only need the objects--not the files.
-	manifests := []client.Object{}
-	for _, m := range capiManifests.RuntimeFiles() {
-		manifests = append(manifests, m.Object)
-	}
-
-	// Run the CAPI system.
-	capiSystem := clusterapi.System()
-	if err := capiSystem.Run(ctx, installConfig); err != nil {
-		return fmt.Errorf("failed to run cluster api system: %w", err)
-	}
-
-	// Grab the client.
-	cl := capiSystem.Client()
-
-	// Create all the manifests and store them.
-	for _, m := range manifests {
-		m.SetNamespace(capiutils.Namespace)
-		if err := cl.Create(context.Background(), m); err != nil {
-			return fmt.Errorf("failed to create manifest: %w", err)
-		}
-		logrus.Infof("Created manifest %+T, namespace=%s name=%s", m, m.GetNamespace(), m.GetName())
-	}
-
-	// Pass cluster kubeconfig and store it in; this is usually the role of a bootstrap provider.
-	{
-		key := client.ObjectKey{
-			Name:      clusterID.InfraID,
-			Namespace: capiutils.Namespace,
-		}
-		cluster := &clusterv1.Cluster{}
-		if err := cl.Get(context.Background(), key, cluster); err != nil {
-			return err
-		}
-		// Create the secret.
-		clusterKubeconfig := clusterKubeconfigAsset.Files()[0].Data
-		secret := utilkubeconfig.GenerateSecret(cluster, clusterKubeconfig)
-		if err := cl.Create(context.Background(), secret); err != nil {
-			return err
-		}
-	}
-
-	// Wait for the load balancer to be ready by checking the control plane endpoint
-	// on the cluster object.
-	var cluster *clusterv1.Cluster
-	{
-		if err := wait.ExponentialBackoff(wait.Backoff{
-			Duration: time.Second * 10,
-			Factor:   float64(1.5),
-			Steps:    32,
-		}, func() (bool, error) {
-			c := &clusterv1.Cluster{}
-			if err := cl.Get(context.Background(), client.ObjectKey{
-				Name:      clusterID.InfraID,
-				Namespace: capiutils.Namespace,
-			}, c); err != nil {
-				if apierrors.IsNotFound(err) {
-					return false, nil
-				}
-				return false, err
-			}
-			cluster = c
-			return cluster.Spec.ControlPlaneEndpoint.IsValid(), nil
-		}); err != nil {
-			return err
-		}
-		if cluster == nil {
-			return errors.New("error occurred during load balancer ready check")
-		}
-		if cluster.Spec.ControlPlaneEndpoint.Host == "" {
-			return errors.New("control plane endpoint is not set")
-		}
-	}
-
-	// Run the post-provisioning steps for the platform we're on.
-	// TODO(vincepri): The following should probably be in a separate package with a clear
-	// interface and multiple hooks at different stages of the cluster lifecycle.
-	switch installConfig.Config.Platform.Name() {
-	case typesaws.Name:
-		ssn, err := installConfig.AWS.Session(context.TODO())
-		if err != nil {
-			return fmt.Errorf("failed to create session: %w", err)
-		}
-		client := awsconfig.NewClient(ssn)
-		r53cfg := awsconfig.GetR53ClientCfg(ssn, "")
-		err = client.CreateOrUpdateRecord(installConfig.Config, cluster.Spec.ControlPlaneEndpoint.Host, r53cfg)
-		if err != nil {
-			return fmt.Errorf("failed to create route53 records: %w", err)
-		}
-		logrus.Infof("Created Route53 records to control plane load balancer.")
-	default:
-	}
-
-	// For each manifest we created, retrieve it and store it in the asset.
-	for _, m := range manifests {
-		key := client.ObjectKey{
-			Name:      m.GetName(),
-			Namespace: m.GetNamespace(),
-		}
-		if err := cl.Get(context.Background(), key, m); err != nil {
-			return fmt.Errorf("failed to get manifest: %w", err)
-		}
-
-		gvk, err := cl.GroupVersionKindFor(m)
-		if err != nil {
-			return fmt.Errorf("failed to get GVK for manifest: %w", err)
-		}
-		fileName := fmt.Sprintf("%s-%s-%s.yaml", gvk.Kind, m.GetNamespace(), m.GetName())
-		objData, err := yaml.Marshal(m)
-		if err != nil {
-			errMsg := fmt.Sprintf("failed to create infrastructure manifest %s from InstallConfig", fileName)
-			return errors.Wrapf(err, errMsg)
-		}
-		c.FileList = append(c.FileList, &asset.File{
-			Filename: fileName,
-			Data:     objData,
-		})
-	}
-
-	logrus.Infof("Cluster API resources have been created. Waiting for cluster to become ready...")
-	return nil
-}
-
 // Files returns the FileList generated by the asset.
 func (c *Cluster) Files() []*asset.File {
 	return c.FileList

pkg/asset/manifests/clusterapi/cluster.go

@@ -14,8 +14,6 @@ import (
 	"sigs.k8s.io/yaml"
 
 	"github.com/openshift/installer/pkg/asset"
-	"github.com/openshift/installer/pkg/asset/ignition/bootstrap"
-	"github.com/openshift/installer/pkg/asset/ignition/machine"
 	"github.com/openshift/installer/pkg/asset/installconfig"
 	"github.com/openshift/installer/pkg/asset/manifests"
 	"github.com/openshift/installer/pkg/asset/manifests/aws"
@@ -49,8 +47,6 @@ func (c *Cluster) Dependencies() []asset.Asset {
 		&installconfig.ClusterID{},
 		&openshiftinstall.Config{},
 		&manifests.FeatureGate{},
-		&bootstrap.Bootstrap{},
-		&machine.Master{},
 		new(rhcos.Image),
 	}
 }
@@ -61,10 +57,8 @@ func (c *Cluster) Generate(dependencies asset.Parents) error {
 	clusterID := &installconfig.ClusterID{}
 	openshiftInstall := &openshiftinstall.Config{}
 	featureGate := &manifests.FeatureGate{}
-	bootstrapIgnAsset := &bootstrap.Bootstrap{}
-	masterIgnAsset := &machine.Master{}
 	rhcosImage := new(rhcos.Image)
-	dependencies.Get(installConfig, clusterID, openshiftInstall, bootstrapIgnAsset, masterIgnAsset, featureGate, rhcosImage)
+	dependencies.Get(installConfig, clusterID, openshiftInstall, featureGate, rhcosImage)
 
 	// If the feature gate is not enabled, do not generate any manifests.
 	if !capiutils.IsEnabled(installConfig) {
@@ -93,49 +87,6 @@ func (c *Cluster) Generate(dependencies asset.Parents) error {
 	}
 	c.FileList = append(c.FileList, &asset.RuntimeFile{Object: cluster, File: asset.File{Filename: "01_capi-cluster.yaml"}})
 
-	// Gather the ignition files, and store them in a secret.
-	{
-		masterIgn := string(masterIgnAsset.Files()[0].Data)
-		bootstrapIgn, err := injectInstallInfo(bootstrapIgnAsset.Files()[0].Data)
-		if err != nil {
-			return errors.Wrap(err, "unable to inject installation info")
-		}
-		c.FileList = append(c.FileList,
-			&asset.RuntimeFile{
-				File: asset.File{Filename: "01_ignition-secret-master.yaml"},
-				Object: &corev1.Secret{
-					ObjectMeta: metav1.ObjectMeta{
-						Name:      fmt.Sprintf("%s-%s", clusterID.InfraID, "master"),
-						Namespace: capiutils.Namespace,
-						Labels: map[string]string{
-							"cluster.x-k8s.io/cluster-name": clusterID.InfraID,
-						},
-					},
-					Data: map[string][]byte{
-						"format": []byte("ignition"),
-						"value":  []byte(masterIgn),
-					},
-				},
-			},
-			&asset.RuntimeFile{
-				File: asset.File{Filename: "01_ignition-secret-bootstrap.yaml"},
-				Object: &corev1.Secret{
-					ObjectMeta: metav1.ObjectMeta{
-						Name:      fmt.Sprintf("%s-%s", clusterID.InfraID, "bootstrap"),
-						Namespace: capiutils.Namespace,
-						Labels: map[string]string{
-							"cluster.x-k8s.io/cluster-name": clusterID.InfraID,
-						},
-					},
-					Data: map[string][]byte{
-						"format": []byte("ignition"),
-						"value":  []byte(bootstrapIgn),
-					},
-				},
-			},
-		)
-	}
-
 	var out *capiutils.GenerateClusterAssetsOutput
 	switch platform := installConfig.Config.Platform.Name(); platform {
 	case awstypes.Name: