Merge pull request #8248 from barbacbd/handle-presigned-url

OCPBUGS-32133: GCP: Fixing GCP Bootstraping

Author: openshift-merge-bot[bot]

data/data/gcp/bootstrap/main.tf

@@ -11,6 +11,37 @@ provider "google" {
   region      = var.gcp_region
 }
 
+resource "google_storage_bucket" "ignition" {
+  name                        = "${var.cluster_id}-bootstrap-ignition"
+  location                    = var.gcp_region
+  uniform_bucket_level_access = true
+  labels                      = var.gcp_extra_labels
+}
+
+resource "google_tags_location_tag_binding" "user_tag_binding_bucket" {
+  for_each = var.gcp_extra_tags
+
+  parent = format("//storage.googleapis.com/projects/_/buckets/%s",
+    google_storage_bucket.ignition.name,
+  )
+  tag_value = each.value
+  location  = var.gcp_region
+
+  depends_on = [google_storage_bucket.ignition]
+}
+
+resource "google_storage_bucket_object" "ignition" {
+  bucket  = google_storage_bucket.ignition.name
+  name    = "bootstrap.ign"
+  content = var.ignition_bootstrap
+}
+
+data "ignition_config" "redirect" {
+  replace {
+    source = var.gcp_signed_url
+  }
+}
+
 resource "google_compute_address" "bootstrap" {
   name        = "${var.cluster_id}-bootstrap-ip"
   description = local.description
@@ -87,7 +118,7 @@ resource "google_compute_instance" "bootstrap" {
   }
 
   metadata = {
-    user-data = var.gcp_ignition_shim
+    user-data = data.ignition_config.redirect.rendered
   }
 
   tags = ["${var.cluster_id}-master", "${var.cluster_id}-bootstrap"]

data/data/gcp/variables-gcp.tf

@@ -175,3 +175,8 @@ variable "gcp_ignition_shim" {
   description = "Ignition stub containing the signed url that points to the bucket containing the ignition data."
   default = ""
 }
+
+variable "gcp_signed_url" {
+  type = string
+  description = "Presigned url for bootstrap ignition link to the bucket where the ignition shim is stored."
+}

pkg/asset/cluster/tfvars/tfvars.go

@@ -519,22 +519,12 @@ func (t *TerraformVariables) Generate(parents asset.Parents) error {
 		ctx, cancel := context.WithTimeout(context.TODO(), 60*time.Second)
 		defer cancel()
 
-		bucketName := gcpbootstrap.GetBootstrapStorageName(clusterID.InfraID)
-		bucketHandle, err := gcpbootstrap.CreateBucketHandle(ctx, bucketName)
-		if err != nil {
-			return fmt.Errorf("failed to create bucket handle %s: %w", bucketName, err)
-		}
-
-		bootstrapIgnURL, err := gcpbootstrap.ProvisionBootstrapStorage(ctx, installConfig, bucketHandle, clusterID.InfraID)
+		url, err := gcpbootstrap.CreateSignedURL(clusterID.InfraID)
 		if err != nil {
 			return fmt.Errorf("failed to provision gcp bootstrap storage resources: %w", err)
 		}
 
-		if err := gcpbootstrap.FillBucket(ctx, bucketHandle, bootstrapIgn); err != nil {
-			return fmt.Errorf("failed to fill bootstrap ignition bucket: %w", err)
-		}
-
-		shim, err := bootstrap.GenerateIgnitionShimWithCertBundleAndProxy(bootstrapIgnURL, installConfig.Config.AdditionalTrustBundle, installConfig.Config.Proxy)
+		shim, err := bootstrap.GenerateIgnitionShimWithCertBundleAndProxy(url, installConfig.Config.AdditionalTrustBundle, installConfig.Config.Proxy)
 		if err != nil {
 			return fmt.Errorf("failed to create gcp ignition shim: %w", err)
 		}
@@ -575,6 +565,7 @@ func (t *TerraformVariables) Generate(parents asset.Parents) error {
 				UserProvisionedDNS:  installConfig.Config.GCP.UserProvisionedDNS == gcp.UserProvisionedDNSEnabled,
 				UserTags:            tags,
 				IgnitionShim:        string(shim),
+				PresignedURL:        url,
 			},
 		)
 		if err != nil {

pkg/asset/ignition/bootstrap/gcp/storage.go

@@ -77,7 +77,13 @@ func CreateStorage(ctx context.Context, ic *installconfig.InstallConfig, bucketH
 }
 
 // CreateSignedURL creates a signed url and correlates the signed url with a storage bucket.
-func CreateSignedURL(handle *storage.BucketHandle, objectName string) (string, error) {
+func CreateSignedURL(clusterID string) (string, error) {
+	bucketName := GetBootstrapStorageName(clusterID)
+	handle, err := CreateBucketHandle(context.Background(), bucketName)
+	if err != nil {
+		return "", fmt.Errorf("creating presigned url, failed to create bucket handle: %w", err)
+	}
+
 	// Signing a URL requires credentials authorized to sign a URL. You can pass
 	// these in through SignedURLOptions with a Google Access ID with
 	// iam.serviceAccounts.signBlob permissions.
@@ -89,44 +95,45 @@ func CreateSignedURL(handle *storage.BucketHandle, objectName string) (string, e
 
 	// The object has not been created yet. This is ok, it is expected to be created after this call.
 	// However, if the object is never created this could cause major issues.
-	url, err := handle.SignedURL(objectName, &opts)
+	url, err := handle.SignedURL(bootstrapIgnitionBucketObjName, &opts)
 	if err != nil {
 		return "", fmt.Errorf("failed to create a signed url: %w", err)
 	}
 
 	return url, nil
 }
 
-// ProvisionBootstrapStorage will provision the required storage bucket and signed url for the bootstrap process.
-func ProvisionBootstrapStorage(ctx context.Context, ic *installconfig.InstallConfig, bucketHandle *storage.BucketHandle, clusterID string) (string, error) {
+// FillBucket will add the contents to the bootstrap storage bucket object.
+func FillBucket(ctx context.Context, bucketHandle *storage.BucketHandle, contents string) error {
 	ctx, cancel := context.WithTimeout(ctx, time.Minute*1)
 	defer cancel()
 
-	if err := CreateStorage(ctx, ic, bucketHandle, clusterID); err != nil {
-		return "", fmt.Errorf("failed to create storage: %w", err)
+	objWriter := bucketHandle.Object(bootstrapIgnitionBucketObjName).NewWriter(ctx)
+	if _, err := fmt.Fprint(objWriter, contents); err != nil {
+		return fmt.Errorf("failed to store content in bucket object: %w", err)
 	}
 
-	url, err := CreateSignedURL(bucketHandle, bootstrapIgnitionBucketObjName)
-	if err != nil {
-		return "", fmt.Errorf("failed to sign url: %w", err)
+	if err := objWriter.Close(); err != nil {
+		return fmt.Errorf("failed to close bucket object writer: %w", err)
 	}
 
-	return url, nil
+	return nil
 }
 
-// FillBucket will add the contents to the bootstrap storage bucket object.
-func FillBucket(ctx context.Context, bucketHandle *storage.BucketHandle, contents string) error {
+// DestroyStorage Destroy the bucket and the bucket objects that are associated with the bucket.
+func DestroyStorage(ctx context.Context, clusterID string) error {
 	ctx, cancel := context.WithTimeout(ctx, time.Minute*1)
 	defer cancel()
 
-	objWriter := bucketHandle.Object(bootstrapIgnitionBucketObjName).NewWriter(ctx)
-	if _, err := fmt.Fprint(objWriter, contents); err != nil {
-		return fmt.Errorf("failed to store content in bucket object: %w", err)
+	client, err := NewStorageClient(ctx)
+	if err != nil {
+		return fmt.Errorf("failed to create storage client: %w", err)
 	}
+	bucketName := GetBootstrapStorageName(clusterID)
 
-	if err := objWriter.Close(); err != nil {
-		return fmt.Errorf("failed to close bucket object writer: %w", err)
+	// Deleting a bucket will delete the managed folders and bucket objects.
+	if err := client.Bucket(bucketName).Delete(ctx); err != nil {
+		return fmt.Errorf("failed to delete bucket %s: %w", bucketName, err)
 	}
-
 	return nil
 }

pkg/infrastructure/clusterapi/clusterapi.go

@@ -20,6 +20,7 @@ import (
 
 	"github.com/openshift/installer/pkg/asset"
 	"github.com/openshift/installer/pkg/asset/cluster/metadata"
+	"github.com/openshift/installer/pkg/asset/cluster/tfvars"
 	"github.com/openshift/installer/pkg/asset/ignition/bootstrap"
 	"github.com/openshift/installer/pkg/asset/ignition/machine"
 	"github.com/openshift/installer/pkg/asset/installconfig"
@@ -66,6 +67,7 @@ func (i *InfraProvider) Provision(ctx context.Context, dir string, parents asset
 	rhcosImage := new(rhcos.Image)
 	bootstrapIgnAsset := &bootstrap.Bootstrap{}
 	masterIgnAsset := &machine.Master{}
+	tfvarsAsset := &tfvars.TerraformVariables{}
 	parents.Get(
 		manifestsAsset,
 		workersAsset,
@@ -77,6 +79,7 @@ func (i *InfraProvider) Provision(ctx context.Context, dir string, parents asset
 		bootstrapIgnAsset,
 		masterIgnAsset,
 		capiMachinesAsset,
+		tfvarsAsset,
 	)
 
 	fileList := []*asset.File{}
@@ -207,6 +210,7 @@ func (i *InfraProvider) Provision(ctx context.Context, dir string, parents asset
 			BootstrapIgnData: bootstrapIgnData,
 			InfraID:          clusterID.InfraID,
 			InstallConfig:    installConfig,
+			TFVarsAsset:      tfvarsAsset,
 		}
 
 		if bootstrapIgnData, err = p.Ignition(ctx, ignInput); err != nil {

pkg/infrastructure/clusterapi/types.go

@@ -5,6 +5,7 @@ import (
 
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
+	"github.com/openshift/installer/pkg/asset/cluster/tfvars"
 	"github.com/openshift/installer/pkg/asset/installconfig"
 	"github.com/openshift/installer/pkg/asset/machines"
 	"github.com/openshift/installer/pkg/asset/manifests"
@@ -51,6 +52,7 @@ type IgnitionInput struct {
 	BootstrapIgnData []byte
 	InfraID          string
 	InstallConfig    *installconfig.InstallConfig
+	TFVarsAsset      *tfvars.TerraformVariables
 }
 
 // InfraReadyProvider defines the InfraReady hook, which is

pkg/infrastructure/gcp/clusterapi/clusterapi.go

@@ -2,14 +2,16 @@ package clusterapi
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"time"
 
 	"github.com/sirupsen/logrus"
 	capg "sigs.k8s.io/cluster-api-provider-gcp/api/v1beta1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
-	"github.com/openshift/installer/pkg/asset/ignition/bootstrap"
+	"github.com/openshift/installer/pkg/asset/cluster/metadata"
+	"github.com/openshift/installer/pkg/asset/cluster/tfvars"
 	"github.com/openshift/installer/pkg/asset/ignition/bootstrap/gcp"
 	"github.com/openshift/installer/pkg/asset/manifests/capiutils"
 	"github.com/openshift/installer/pkg/infrastructure/clusterapi"
@@ -80,10 +82,10 @@ func (p Provider) Ignition(ctx context.Context, in clusterapi.IgnitionInput) ([]
 		return nil, fmt.Errorf("failed to create bucket handle %s: %w", bucketName, err)
 	}
 
-	url, err := gcp.ProvisionBootstrapStorage(ctx, in.InstallConfig, bucketHandle, in.InfraID)
-	if err != nil {
-		return nil, fmt.Errorf("ignition failed to provision storage: %w", err)
+	if err := gcp.CreateStorage(ctx, in.InstallConfig, bucketHandle, in.InfraID); err != nil {
+		return nil, fmt.Errorf("failed to create bucket %s: %w", bucketName, err)
 	}
+
 	editedIgnitionBytes, err := EditIgnition(ctx, in)
 	if err != nil {
 		return nil, fmt.Errorf("failed to edit bootstrap ignition: %w", err)
@@ -98,12 +100,25 @@ func (p Provider) Ignition(ctx context.Context, in clusterapi.IgnitionInput) ([]
 		return nil, fmt.Errorf("ignition failed to fill bucket: %w", err)
 	}
 
-	ignShim, err := bootstrap.GenerateIgnitionShimWithCertBundleAndProxy(url, in.InstallConfig.Config.AdditionalTrustBundle, in.InstallConfig.Config.Proxy)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create ignition shim: %w", err)
+	for _, file := range in.TFVarsAsset.Files() {
+		if file.Filename == tfvars.TfPlatformVarsFileName {
+			var found bool
+			tfvarsData := make(map[string]interface{})
+			err = json.Unmarshal(file.Data, &tfvarsData)
+			if err != nil {
+				return nil, fmt.Errorf("failed to unmarshal %s to json: %w", tfvars.TfPlatformVarsFileName, err)
+			}
+
+			ignShim, found := tfvarsData["gcp_ignition_shim"].(string)
+			if !found {
+				return nil, fmt.Errorf("failed to find ignition shim")
+			}
+
+			return []byte(ignShim), nil
+		}
 	}
 
-	return ignShim, nil
+	return nil, fmt.Errorf("failed to complete ignition process")
 }
 
 // InfraReady is called once cluster.Status.InfrastructureReady
@@ -163,6 +178,20 @@ func (p Provider) InfraReady(ctx context.Context, in clusterapi.InfraReadyInput)
 	return nil
 }
 
+// DestroyBootstrap destroys the temporary bootstrap resources.
+func (p Provider) DestroyBootstrap(dir string) error {
+	logrus.Warnf("Destroying GCP Bootstrap Resources")
+	metadata, err := metadata.Load(dir)
+	if err != nil {
+		return err
+	}
+
+	if err := gcp.DestroyStorage(context.Background(), metadata.ClusterID); err != nil {
+		return fmt.Errorf("failed to destroy storage")
+	}
+	return nil
+}
+
 // PostProvision should be called to add or update and GCP resources after provisioning has completed.
 func (p Provider) PostProvision(ctx context.Context, in clusterapi.PostProvisionInput) error {
 	return nil

pkg/terraform/stages/gcp/stages.go

@@ -1,18 +1,16 @@
 package gcp
 
 import (
-	"context"
 	"encoding/json"
 	"fmt"
-	"time"
+	"os"
 
 	igntypes "github.com/coreos/ignition/v2/config/v3_2/types"
 	"github.com/hashicorp/terraform-exec/tfexec"
 	"github.com/pkg/errors"
 
 	"github.com/openshift/installer/pkg/asset"
 	"github.com/openshift/installer/pkg/asset/ignition"
-	gcpbootstrap "github.com/openshift/installer/pkg/asset/ignition/bootstrap/gcp"
 	"github.com/openshift/installer/pkg/asset/lbconfig"
 	"github.com/openshift/installer/pkg/terraform"
 	"github.com/openshift/installer/pkg/terraform/providers"
@@ -120,21 +118,20 @@ func extractGCPLBConfig(s stages.SplitStage, directory string, terraformDir stri
 		return "", err
 	}
 
-	clusterID, ok := tfvarData["cluster_id"]
-	if !ok {
-		return "", fmt.Errorf("failed to read cluster id from tfvars")
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), time.Minute*1)
-	defer cancel()
+	// Update the ignition bootstrap variable to include the lbconfig.
+	tfvarData["ignition_bootstrap"] = string(ignitionOutput)
 
-	bucketName := gcpbootstrap.GetBootstrapStorageName(clusterID.(string))
-	bucketHandle, err := gcpbootstrap.CreateBucketHandle(ctx, bucketName)
+	// Convert the bootstrap data and write the data back to a file. This will overwrite the original tfvars file.
+	jsonBootstrap, err := json.Marshal(tfvarData)
 	if err != nil {
-		return "", fmt.Errorf("failed to create bucket handle %s: %w", bucketName, err)
+		return "", fmt.Errorf("failed to convert bootstrap ignition to bytes: %w", err)
 	}
-	if err := gcpbootstrap.FillBucket(context.Background(), bucketHandle, string(ignitionOutput)); err != nil {
-		return "", fmt.Errorf("failed to fill gcp bucket with updated boostrap ignition contents: %w", err)
+	tfvarsFile.Data = jsonBootstrap
+
+	// update the value on disk to match
+	if err := os.WriteFile(fmt.Sprintf("%s/%s", directory, tfvarsFile.Filename), jsonBootstrap, 0o600); err != nil {
+		return "", fmt.Errorf("failed to rewrite %s: %w", tfvarsFile.Filename, err)
 	}
+
 	return "", nil
 }

pkg/tfvars/gcp/gcp.go

@@ -47,6 +47,7 @@ type config struct {
 	UserProvisionedDNS        bool              `json:"gcp_user_provisioned_dns,omitempty"`
 	ExtraTags                 map[string]string `json:"gcp_extra_tags,omitempty"`
 	IgnitionShim              string            `json:"gcp_ignition_shim,omitempty"`
+	PresignedURL              string            `json:"gcp_signed_url"`
 }
 
 // TFVarsSources contains the parameters to be converted into Terraform variables
@@ -63,6 +64,7 @@ type TFVarsSources struct {
 	UserProvisionedDNS  bool
 	UserTags            map[string]string
 	IgnitionShim        string
+	PresignedURL        string
 }
 
 // TFVars generates gcp-specific Terraform variables launching the cluster.
@@ -106,6 +108,7 @@ func TFVars(sources TFVarsSources) ([]byte, error) {
 		UserProvisionedDNS:        sources.UserProvisionedDNS,
 		ExtraTags:                 sources.UserTags,
 		IgnitionShim:              sources.IgnitionShim,
+		PresignedURL:              sources.PresignedURL,
 	}
 
 	if masterConfig.Disks[0].EncryptionKey != nil {