openstack: Run preprovision steps for CAPI

Co-Authored-By: Pierre Prinetti <[email protected]>
Co-Authored-By: Martin André <[email protected]>

Author: 3 people

pkg/asset/cluster/cluster.go

@@ -61,9 +61,11 @@ func (c *Cluster) Dependencies() []asset.Asset {
 		&installconfig.PlatformCredsCheck{},
 		&installconfig.PlatformPermsCheck{},
 		&installconfig.PlatformProvisionCheck{},
+		new(rhcos.Image),
 		&quota.PlatformQuotaCheck{},
 		&tfvars.TerraformVariables{},
 		&password.KubeadminPassword{},
+		&manifests.Manifests{},
 		&capimanifests.Cluster{},
 		&kubeconfig.AdminClient{},
 		&bootstrap.Bootstrap{},
@@ -82,8 +84,9 @@ func (c *Cluster) Generate(parents asset.Parents) (err error) {
 
 	clusterID := &installconfig.ClusterID{}
 	installConfig := &installconfig.InstallConfig{}
+	rhcosImage := new(rhcos.Image)
 	terraformVariables := &tfvars.TerraformVariables{}
-	parents.Get(clusterID, installConfig, terraformVariables)
+	parents.Get(clusterID, installConfig, terraformVariables, rhcosImage)
 
 	if fs := installConfig.Config.FeatureSet; strings.HasSuffix(string(fs), "NoUpgrade") {
 		logrus.Warnf("FeatureSet %q is enabled. This FeatureSet does not allow upgrades and may affect the supportability of the cluster.", fs)
@@ -116,7 +119,14 @@ func (c *Cluster) Generate(parents asset.Parents) (err error) {
 			return err
 		}
 	case typesopenstack.Name:
-		if err := openstack.PreTerraform(); err != nil {
+		var tfvarsFile *asset.File
+		for _, f := range terraformVariables.Files() {
+			if f.Filename == tfvars.TfPlatformVarsFileName {
+				tfvarsFile = f
+				break
+			}
+		}
+		if err := openstack.PreTerraform(context.TODO(), tfvarsFile, installConfig, clusterID, rhcosImage); err != nil {
 			return err
 		}
 	}

pkg/asset/cluster/openstack/openstack.go

@@ -3,42 +3,18 @@
 package openstack
 
 import (
-	"errors"
-	"fmt"
-	"os"
-	"path/filepath"
+	"context"
 
+	"github.com/openshift/installer/pkg/asset"
+	"github.com/openshift/installer/pkg/asset/installconfig"
+	"github.com/openshift/installer/pkg/asset/manifests/capiutils"
+	rhcos_asset "github.com/openshift/installer/pkg/asset/rhcos"
+	"github.com/openshift/installer/pkg/infrastructure/openstack/preprovision"
+	"github.com/openshift/installer/pkg/rhcos"
 	"github.com/openshift/installer/pkg/types"
 	"github.com/openshift/installer/pkg/types/openstack"
 )
 
-// PreTerraform performs any infrastructure initialization which must
-// happen before Terraform creates the remaining infrastructure.
-func PreTerraform() error {
-	// Terraform runs in a different directory but we want to allow people to
-	// use clouds.yaml files in their local directory. Emulate this by setting
-	// the necessary environment variable to point to this file if (a) the user
-	// hasn't already set this environment variable and (b) there is actually
-	// a local file
-	if path := os.Getenv("OS_CLIENT_CONFIG_FILE"); path != "" {
-		return nil
-	}
-
-	cwd, err := os.Getwd()
-	if err != nil {
-		return fmt.Errorf("unable to determine working directory: %w", err)
-	}
-
-	cloudsYAML := filepath.Join(cwd, "clouds.yaml")
-	if _, err = os.Stat(cloudsYAML); err == nil {
-		os.Setenv("OS_CLIENT_CONFIG_FILE", cloudsYAML)
-	} else if !errors.Is(err, os.ErrNotExist) {
-		return fmt.Errorf("unable to determine if clouds.yaml exists: %w", err)
-	}
-
-	return nil
-}
-
 // Metadata converts an install configuration to OpenStack metadata.
 func Metadata(infraID string, config *types.InstallConfig) *openstack.Metadata {
 	return &openstack.Metadata{
@@ -48,3 +24,28 @@ func Metadata(infraID string, config *types.InstallConfig) *openstack.Metadata {
 		},
 	}
 }
+
+// PreTerraform performs any infrastructure initialization which must
+// happen before Terraform creates the remaining infrastructure.
+func PreTerraform(ctx context.Context, tfvarsFile *asset.File, installConfig *installconfig.InstallConfig, clusterID *installconfig.ClusterID, rhcosImage *rhcos_asset.Image) error {
+	if !capiutils.IsEnabled(installConfig) {
+		if err := preprovision.ReplaceBootstrapIgnitionInTFVars(ctx, tfvarsFile, installConfig, clusterID); err != nil {
+			return err
+		}
+
+		if err := preprovision.TagVIPPorts(ctx, installConfig, clusterID.InfraID); err != nil {
+			return err
+		}
+
+		// upload the corresponding image to Glance if rhcosImage contains a
+		// URL. If rhcosImage contains a name, then that points to an existing
+		// Glance image.
+		if imageName, isURL := rhcos.GenerateOpenStackImageName(string(*rhcosImage), clusterID.InfraID); isURL {
+			if err := preprovision.UploadBaseImage(ctx, installConfig.Config.Platform.OpenStack.Cloud, string(*rhcosImage), imageName, clusterID.InfraID, installConfig.Config.Platform.OpenStack.ClusterOSImageProperties); err != nil {
+				return err
+			}
+		}
+	}
+
+	return preprovision.SetTerraformEnvironment()
+}

pkg/destroy/bootstrap/bootstrap.go

@@ -12,8 +12,8 @@ import (
 
 	configv1 "github.com/openshift/api/config/v1"
 	"github.com/openshift/installer/pkg/asset/cluster/metadata"
-	openstackasset "github.com/openshift/installer/pkg/asset/cluster/openstack"
 	osp "github.com/openshift/installer/pkg/destroy/openstack"
+	"github.com/openshift/installer/pkg/infrastructure/openstack/preprovision"
 	infra "github.com/openshift/installer/pkg/infrastructure/platform"
 	ibmcloudtfvars "github.com/openshift/installer/pkg/tfvars/ibmcloud"
 	typesazure "github.com/openshift/installer/pkg/types/azure"
@@ -35,7 +35,7 @@ func Destroy(ctx context.Context, dir string) (err error) {
 	}
 
 	if platform == openstack.Name {
-		if err := openstackasset.PreTerraform(); err != nil {
+		if err := preprovision.SetTerraformEnvironment(); err != nil {
 			return errors.Wrapf(err, "Failed to  initialize infrastructure")
 		}
 

pkg/infrastructure/openstack/clusterapi/clusterapi.go

@@ -4,15 +4,22 @@ import (
 	"context"
 	"fmt"
 
-	capo "sigs.k8s.io/cluster-api-provider-openstack/api/v1alpha7"
-	"sigs.k8s.io/controller-runtime/pkg/client"
-
 	"github.com/gophercloud/gophercloud"
 	"github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/attributestags"
 	"github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/layer3/floatingips"
 	"github.com/gophercloud/gophercloud/openstack/networking/v2/ports"
+	"github.com/sirupsen/logrus"
+	"gopkg.in/yaml.v2"
+	capo "sigs.k8s.io/cluster-api-provider-openstack/api/v1alpha7"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	configv1 "github.com/openshift/api/config/v1"
+	"github.com/openshift/installer/pkg/asset/manifests"
 	"github.com/openshift/installer/pkg/asset/manifests/capiutils"
 	"github.com/openshift/installer/pkg/infrastructure/clusterapi"
+	"github.com/openshift/installer/pkg/infrastructure/openstack/preprovision"
+	"github.com/openshift/installer/pkg/rhcos"
+	"github.com/openshift/installer/pkg/types/openstack"
 	openstackdefaults "github.com/openshift/installer/pkg/types/openstack/defaults"
 )
 
@@ -21,42 +28,89 @@ type Provider struct {
 	clusterapi.InfraProvider
 }
 
-// func (p Provider) PreProvision(in clusterapi.PreProvisionInput) error {
-// 	return nil
-// }
+// Name contains the name of the openstack provider.
+func (p Provider) Name() string {
+	return openstack.Name
+}
+
+var _ clusterapi.PreProvider = Provider{}
+
+// PreProvision tags the VIP ports and creates the security groups that are needed during CAPI provisioning.
+func (p Provider) PreProvision(ctx context.Context, in clusterapi.PreProvisionInput) error {
+	var (
+		infraID            = in.InfraID
+		installConfig      = in.InstallConfig
+		rhcosImage         = string(*in.RhcosImage)
+		mastersSchedulable bool
+	)
+
+	for _, f := range in.ManifestsAsset.Files() {
+		if f.Filename == manifests.SchedulerCfgFilename {
+			schedulerConfig := configv1.Scheduler{}
+			if err := yaml.Unmarshal(f.Data, &schedulerConfig); err != nil {
+				return fmt.Errorf("unable to decode the scheduler manifest: %w", err)
+			}
+			mastersSchedulable = schedulerConfig.Spec.MastersSchedulable
+			break
+		}
+	}
+
+	if err := preprovision.TagVIPPorts(ctx, installConfig, infraID); err != nil {
+		return err
+	}
+
+	// upload the corresponding image to Glance if rhcosImage contains a
+	// URL. If rhcosImage contains a name, then that points to an existing
+	// Glance image.
+	if imageName, isURL := rhcos.GenerateOpenStackImageName(rhcosImage, infraID); isURL {
+		if err := preprovision.UploadBaseImage(ctx, installConfig.Config.Platform.OpenStack.Cloud, rhcosImage, imageName, infraID, installConfig.Config.Platform.OpenStack.ClusterOSImageProperties); err != nil {
+			return err
+		}
+	}
+
+	return preprovision.SecurityGroups(ctx, installConfig, infraID, mastersSchedulable)
+}
+
+var _ clusterapi.InfraReadyProvider = Provider{}
 
-func (p Provider) ControlPlaneAvailable(in clusterapi.ControlPlaneAvailableInput) error {
+// InfraReady creates the API and Ingress ports and attaches the Floating IPs to them.
+func (p Provider) InfraReady(ctx context.Context, in clusterapi.InfraReadyInput) error {
+	var (
+		k8sClient     = in.Client
+		infraID       = in.InfraID
+		installConfig = in.InstallConfig
+	)
 	ospCluster := &capo.OpenStackCluster{}
 	key := client.ObjectKey{
-		Name:      in.InfraID,
+		Name:      infraID,
 		Namespace: capiutils.Namespace,
 	}
-	if err := in.Client.Get(context.Background(), key, ospCluster); err != nil {
+	if err := k8sClient.Get(ctx, key, ospCluster); err != nil {
 		return fmt.Errorf("failed to get OSPCluster: %w", err)
 	}
 
-	networkClient, err := openstackdefaults.NewServiceClient("network", openstackdefaults.DefaultClientOpts(in.InstallConfig.Config.Platform.OpenStack.Cloud))
+	networkClient, err := openstackdefaults.NewServiceClient("network", openstackdefaults.DefaultClientOpts(installConfig.Config.Platform.OpenStack.Cloud))
 	if err != nil {
 		return err
 	}
 
-	apiPort, err := createPort(networkClient, "api", in.InfraID, ospCluster.Status.Network.ID)
+	apiPort, err := createPort(networkClient, "api", infraID, ospCluster.Status.Network.ID, ospCluster.Status.Network.Subnets[0].ID, installConfig.Config.Platform.OpenStack.APIVIPs[0])
 	if err != nil {
 		return err
 	}
-	if in.InstallConfig.Config.OpenStack.APIFloatingIP != "" {
-		err = assignFIP(networkClient, in.InstallConfig.Config.OpenStack.APIFloatingIP, apiPort)
+	if installConfig.Config.OpenStack.APIFloatingIP != "" {
+		err = assignFIP(networkClient, installConfig.Config.OpenStack.APIFloatingIP, apiPort)
 		if err != nil {
 			return err
 		}
 	}
 
-	ingressPort, err := createPort(networkClient, "ingress", in.InfraID, ospCluster.Status.Network.ID)
+	ingressPort, err := createPort(networkClient, "ingress", infraID, ospCluster.Status.Network.ID, ospCluster.Status.Network.Subnets[0].ID, installConfig.Config.Platform.OpenStack.IngressVIPs[0])
 	if err != nil {
 		return err
 	}
-	if in.InstallConfig.Config.OpenStack.IngressFloatingIP != "" {
-		err = assignFIP(networkClient, in.InstallConfig.Config.OpenStack.IngressFloatingIP, ingressPort)
+	if installConfig.Config.OpenStack.IngressFloatingIP != "" {
+		err = assignFIP(networkClient, installConfig.Config.OpenStack.IngressFloatingIP, ingressPort)
 		if err != nil {
 			return err
 		}
@@ -65,14 +119,33 @@ func (p Provider) ControlPlaneAvailable(in clusterapi.ControlPlaneAvailableInput
 	return nil
 }
 
-func createPort(client *gophercloud.ServiceClient, role, infraID, networkID string) (*ports.Port, error) {
-	createOtps := ports.CreateOpts{
+var _ clusterapi.IgnitionProvider = Provider{}
+
+// Ignition uploads the bootstrap machine's Ignition file to OpenStack.
+func (p Provider) Ignition(ctx context.Context, in clusterapi.IgnitionInput) ([]byte, error) {
+	logrus.Debugf("Uploading the bootstrap machine's Ignition file to OpenStack")
+	var (
+		bootstrapIgnData = in.BootstrapIgnData
+		infraID          = in.InfraID
+		installConfig    = in.InstallConfig
+	)
+
+	return preprovision.UploadIgnitionAndBuildShim(ctx, installConfig.Config.Platform.OpenStack.Cloud, infraID, installConfig.Config.Proxy, bootstrapIgnData)
+}
+
+func createPort(client *gophercloud.ServiceClient, role, infraID, networkID, subnetID, fixedIP string) (*ports.Port, error) {
+	createOpts := ports.CreateOpts{
 		Name:        fmt.Sprintf("%s-%s-port", infraID, role),
 		NetworkID:   networkID,
 		Description: "Created By OpenShift Installer",
+		FixedIPs: []ports.IP{
+			{
+				IPAddress: fixedIP,
+				SubnetID:  subnetID,
+			}},
 	}
 
-	port, err := ports.Create(client, createOtps).Extract()
+	port, err := ports.Create(client, createOpts).Extract()
 	if err != nil {
 		return nil, err
 	}