Skip to content

Commit 2642483

Browse files
committed
GCP: Fixing GCP Bootstraping
** Adjusting the use of a presigned url. Passing to terraform. ** Adding the bucket and bucket object back to terraform ** Presigned url is created for both capg and terraform installs. ** CAPG will create and destroy the bucket and bucket object for bootstrap.
1 parent 9a8b820 commit 2642483

File tree

9 files changed

+123
-54
lines changed

9 files changed

+123
-54
lines changed

data/data/gcp/bootstrap/main.tf

Lines changed: 32 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,37 @@ provider "google" {
1111
region = var.gcp_region
1212
}
1313

14+
resource "google_storage_bucket" "ignition" {
15+
name = "${var.cluster_id}-bootstrap-ignition"
16+
location = var.gcp_region
17+
uniform_bucket_level_access = true
18+
labels = var.gcp_extra_labels
19+
}
20+
21+
resource "google_tags_location_tag_binding" "user_tag_binding_bucket" {
22+
for_each = var.gcp_extra_tags
23+
24+
parent = format("//storage.googleapis.com/projects/_/buckets/%s",
25+
google_storage_bucket.ignition.name,
26+
)
27+
tag_value = each.value
28+
location = var.gcp_region
29+
30+
depends_on = [google_storage_bucket.ignition]
31+
}
32+
33+
resource "google_storage_bucket_object" "ignition" {
34+
bucket = google_storage_bucket.ignition.name
35+
name = "bootstrap.ign"
36+
content = var.ignition_bootstrap
37+
}
38+
39+
data "ignition_config" "redirect" {
40+
replace {
41+
source = var.gcp_signed_url
42+
}
43+
}
44+
1445
resource "google_compute_address" "bootstrap" {
1546
name = "${var.cluster_id}-bootstrap-ip"
1647
description = local.description
@@ -87,7 +118,7 @@ resource "google_compute_instance" "bootstrap" {
87118
}
88119

89120
metadata = {
90-
user-data = var.gcp_ignition_shim
121+
user-data = data.ignition_config.redirect.rendered
91122
}
92123

93124
tags = ["${var.cluster_id}-master", "${var.cluster_id}-bootstrap"]

data/data/gcp/variables-gcp.tf

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -175,3 +175,8 @@ variable "gcp_ignition_shim" {
175175
description = "Ignition stub containing the signed url that points to the bucket containing the ignition data."
176176
default = ""
177177
}
178+
179+
variable "gcp_signed_url" {
180+
type = string
181+
description = "Presigned url for bootstrap ignition link to the bucket where the ignition shim is stored."
182+
}

pkg/asset/cluster/tfvars/tfvars.go

Lines changed: 3 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -519,22 +519,12 @@ func (t *TerraformVariables) Generate(parents asset.Parents) error {
519519
ctx, cancel := context.WithTimeout(context.TODO(), 60*time.Second)
520520
defer cancel()
521521

522-
bucketName := gcpbootstrap.GetBootstrapStorageName(clusterID.InfraID)
523-
bucketHandle, err := gcpbootstrap.CreateBucketHandle(ctx, bucketName)
524-
if err != nil {
525-
return fmt.Errorf("failed to create bucket handle %s: %w", bucketName, err)
526-
}
527-
528-
bootstrapIgnURL, err := gcpbootstrap.ProvisionBootstrapStorage(ctx, installConfig, bucketHandle, clusterID.InfraID)
522+
url, err := gcpbootstrap.CreateSignedURL(clusterID.InfraID)
529523
if err != nil {
530524
return fmt.Errorf("failed to provision gcp bootstrap storage resources: %w", err)
531525
}
532526

533-
if err := gcpbootstrap.FillBucket(ctx, bucketHandle, bootstrapIgn); err != nil {
534-
return fmt.Errorf("failed to fill bootstrap ignition bucket: %w", err)
535-
}
536-
537-
shim, err := bootstrap.GenerateIgnitionShimWithCertBundleAndProxy(bootstrapIgnURL, installConfig.Config.AdditionalTrustBundle, installConfig.Config.Proxy)
527+
shim, err := bootstrap.GenerateIgnitionShimWithCertBundleAndProxy(url, installConfig.Config.AdditionalTrustBundle, installConfig.Config.Proxy)
538528
if err != nil {
539529
return fmt.Errorf("failed to create gcp ignition shim: %w", err)
540530
}
@@ -576,6 +566,7 @@ func (t *TerraformVariables) Generate(parents asset.Parents) error {
576566
UserProvisionedDNS: installConfig.Config.GCP.UserProvisionedDNS == gcp.UserProvisionedDNSEnabled,
577567
UserTags: tags,
578568
IgnitionShim: string(shim),
569+
PresignedURL: url,
579570
},
580571
)
581572
if err != nil {

pkg/asset/ignition/bootstrap/gcp/storage.go

Lines changed: 25 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,13 @@ func CreateStorage(ctx context.Context, ic *installconfig.InstallConfig, bucketH
7676
}
7777

7878
// CreateSignedURL creates a signed url and correlates the signed url with a storage bucket.
79-
func CreateSignedURL(handle *storage.BucketHandle, objectName string) (string, error) {
79+
func CreateSignedURL(clusterID string) (string, error) {
80+
bucketName := GetBootstrapStorageName(clusterID)
81+
handle, err := CreateBucketHandle(context.Background(), bucketName)
82+
if err != nil {
83+
return "", fmt.Errorf("creating presigned url, failed to create bucket handle: %w", err)
84+
}
85+
8086
// Signing a URL requires credentials authorized to sign a URL. You can pass
8187
// these in through SignedURLOptions with a Google Access ID with
8288
// iam.serviceAccounts.signBlob permissions.
@@ -88,44 +94,45 @@ func CreateSignedURL(handle *storage.BucketHandle, objectName string) (string, e
8894

8995
// The object has not been created yet. This is ok, it is expected to be created after this call.
9096
// However, if the object is never created this could cause major issues.
91-
url, err := handle.SignedURL(objectName, &opts)
97+
url, err := handle.SignedURL(bootstrapIgnitionBucketObjName, &opts)
9298
if err != nil {
9399
return "", fmt.Errorf("failed to create a signed url: %w", err)
94100
}
95101

96102
return url, nil
97103
}
98104

99-
// ProvisionBootstrapStorage will provision the required storage bucket and signed url for the bootstrap process.
100-
func ProvisionBootstrapStorage(ctx context.Context, ic *installconfig.InstallConfig, bucketHandle *storage.BucketHandle, clusterID string) (string, error) {
105+
// FillBucket will add the contents to the bootstrap storage bucket object.
106+
func FillBucket(ctx context.Context, bucketHandle *storage.BucketHandle, contents string) error {
101107
ctx, cancel := context.WithTimeout(ctx, time.Minute*1)
102108
defer cancel()
103109

104-
if err := CreateStorage(ctx, ic, bucketHandle, clusterID); err != nil {
105-
return "", fmt.Errorf("failed to create storage: %w", err)
110+
objWriter := bucketHandle.Object(bootstrapIgnitionBucketObjName).NewWriter(ctx)
111+
if _, err := fmt.Fprint(objWriter, contents); err != nil {
112+
return fmt.Errorf("failed to store content in bucket object: %w", err)
106113
}
107114

108-
url, err := CreateSignedURL(bucketHandle, bootstrapIgnitionBucketObjName)
109-
if err != nil {
110-
return "", fmt.Errorf("failed to sign url: %w", err)
115+
if err := objWriter.Close(); err != nil {
116+
return fmt.Errorf("failed to close bucket object writer: %w", err)
111117
}
112118

113-
return url, nil
119+
return nil
114120
}
115121

116-
// FillBucket will add the contents to the bootstrap storage bucket object.
117-
func FillBucket(ctx context.Context, bucketHandle *storage.BucketHandle, contents string) error {
122+
// DestroyStorage Destroy the bucket and the bucket objects that are associated with the bucket.
123+
func DestroyStorage(ctx context.Context, clusterID string) error {
118124
ctx, cancel := context.WithTimeout(ctx, time.Minute*1)
119125
defer cancel()
120126

121-
objWriter := bucketHandle.Object(bootstrapIgnitionBucketObjName).NewWriter(ctx)
122-
if _, err := fmt.Fprint(objWriter, contents); err != nil {
123-
return fmt.Errorf("failed to store content in bucket object: %w", err)
127+
client, err := NewStorageClient(ctx)
128+
if err != nil {
129+
return fmt.Errorf("failed to create storage client: %w", err)
124130
}
131+
bucketName := GetBootstrapStorageName(clusterID)
125132

126-
if err := objWriter.Close(); err != nil {
127-
return fmt.Errorf("failed to close bucket object writer: %w", err)
133+
// Deleting a bucket will delete the managed folders and bucket objects.
134+
if err := client.Bucket(bucketName).Delete(ctx); err != nil {
135+
return fmt.Errorf("failed to delete bucket %s: %w", bucketName, err)
128136
}
129-
130137
return nil
131138
}

pkg/infrastructure/clusterapi/clusterapi.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import (
2121

2222
"github.com/openshift/installer/pkg/asset"
2323
"github.com/openshift/installer/pkg/asset/cluster/metadata"
24+
"github.com/openshift/installer/pkg/asset/cluster/tfvars"
2425
"github.com/openshift/installer/pkg/asset/ignition/bootstrap"
2526
"github.com/openshift/installer/pkg/asset/ignition/machine"
2627
"github.com/openshift/installer/pkg/asset/installconfig"
@@ -67,6 +68,7 @@ func (i *InfraProvider) Provision(dir string, parents asset.Parents) ([]*asset.F
6768
rhcosImage := new(rhcos.Image)
6869
bootstrapIgnAsset := &bootstrap.Bootstrap{}
6970
masterIgnAsset := &machine.Master{}
71+
tfvarsAsset := &tfvars.TerraformVariables{}
7072
parents.Get(
7173
manifestsAsset,
7274
workersAsset,
@@ -78,6 +80,7 @@ func (i *InfraProvider) Provision(dir string, parents asset.Parents) ([]*asset.F
7880
bootstrapIgnAsset,
7981
masterIgnAsset,
8082
capiMachinesAsset,
83+
tfvarsAsset,
8184
)
8285

8386
fileList := []*asset.File{}
@@ -217,6 +220,7 @@ func (i *InfraProvider) Provision(dir string, parents asset.Parents) ([]*asset.F
217220
BootstrapIgnData: bootstrapIgnData,
218221
InfraID: clusterID.InfraID,
219222
InstallConfig: installConfig,
223+
TFVarsAsset: tfvarsAsset,
220224
}
221225

222226
if bootstrapIgnData, err = p.Ignition(ctx, ignInput); err != nil {

pkg/infrastructure/clusterapi/types.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55

66
"sigs.k8s.io/controller-runtime/pkg/client"
77

8+
"github.com/openshift/installer/pkg/asset/cluster/tfvars"
89
"github.com/openshift/installer/pkg/asset/installconfig"
910
"github.com/openshift/installer/pkg/asset/machines"
1011
"github.com/openshift/installer/pkg/asset/manifests"
@@ -51,6 +52,7 @@ type IgnitionInput struct {
5152
BootstrapIgnData []byte
5253
InfraID string
5354
InstallConfig *installconfig.InstallConfig
55+
TFVarsAsset *tfvars.TerraformVariables
5456
}
5557

5658
// InfraReadyProvider defines the InfraReady hook, which is

pkg/infrastructure/gcp/clusterapi/clusterapi.go

Lines changed: 37 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,16 @@ package clusterapi
22

33
import (
44
"context"
5+
"encoding/json"
56
"fmt"
67
"time"
78

89
"github.com/sirupsen/logrus"
910
capg "sigs.k8s.io/cluster-api-provider-gcp/api/v1beta1"
1011
"sigs.k8s.io/controller-runtime/pkg/client"
1112

12-
"github.com/openshift/installer/pkg/asset/ignition/bootstrap"
13+
"github.com/openshift/installer/pkg/asset/cluster/metadata"
14+
"github.com/openshift/installer/pkg/asset/cluster/tfvars"
1315
"github.com/openshift/installer/pkg/asset/ignition/bootstrap/gcp"
1416
"github.com/openshift/installer/pkg/asset/manifests/capiutils"
1517
"github.com/openshift/installer/pkg/infrastructure/clusterapi"
@@ -80,10 +82,10 @@ func (p Provider) Ignition(ctx context.Context, in clusterapi.IgnitionInput) ([]
8082
return nil, fmt.Errorf("failed to create bucket handle %s: %w", bucketName, err)
8183
}
8284

83-
url, err := gcp.ProvisionBootstrapStorage(ctx, in.InstallConfig, bucketHandle, in.InfraID)
84-
if err != nil {
85-
return nil, fmt.Errorf("ignition failed to provision storage: %w", err)
85+
if err := gcp.CreateStorage(ctx, in.InstallConfig, bucketHandle, in.InfraID); err != nil {
86+
return nil, fmt.Errorf("failed to create bucket %s: %w", bucketName, err)
8687
}
88+
8789
editedIgnitionBytes, err := EditIgnition(ctx, in)
8890
if err != nil {
8991
return nil, fmt.Errorf("failed to edit bootstrap ignition: %w", err)
@@ -98,12 +100,25 @@ func (p Provider) Ignition(ctx context.Context, in clusterapi.IgnitionInput) ([]
98100
return nil, fmt.Errorf("ignition failed to fill bucket: %w", err)
99101
}
100102

101-
ignShim, err := bootstrap.GenerateIgnitionShimWithCertBundleAndProxy(url, in.InstallConfig.Config.AdditionalTrustBundle, in.InstallConfig.Config.Proxy)
102-
if err != nil {
103-
return nil, fmt.Errorf("failed to create ignition shim: %w", err)
103+
for _, file := range in.TFVarsAsset.Files() {
104+
if file.Filename == tfvars.TfPlatformVarsFileName {
105+
var found bool
106+
tfvarsData := make(map[string]interface{})
107+
err = json.Unmarshal(file.Data, &tfvarsData)
108+
if err != nil {
109+
return nil, fmt.Errorf("failed to unmarshal %s to json: %w", tfvars.TfPlatformVarsFileName, err)
110+
}
111+
112+
ignShim, found := tfvarsData["gcp_ignition_shim"].(string)
113+
if !found {
114+
return nil, fmt.Errorf("failed to find ignition shim")
115+
}
116+
117+
return []byte(ignShim), nil
118+
}
104119
}
105120

106-
return ignShim, nil
121+
return nil, fmt.Errorf("failed to complete ignition process")
107122
}
108123

109124
// InfraReady is called once cluster.Status.InfrastructureReady
@@ -163,6 +178,20 @@ func (p Provider) InfraReady(ctx context.Context, in clusterapi.InfraReadyInput)
163178
return nil
164179
}
165180

181+
// DestroyBootstrap destroys the temporary bootstrap resources.
182+
func (p Provider) DestroyBootstrap(dir string) error {
183+
logrus.Warnf("Destroying GCP Bootstrap Resources")
184+
metadata, err := metadata.Load(dir)
185+
if err != nil {
186+
return err
187+
}
188+
189+
if err := gcp.DestroyStorage(context.Background(), metadata.ClusterID); err != nil {
190+
return fmt.Errorf("failed to destroy storage")
191+
}
192+
return nil
193+
}
194+
166195
// PostProvision should be called to add or update and GCP resources after provisioning has completed.
167196
func (p Provider) PostProvision(ctx context.Context, in clusterapi.PostProvisionInput) error {
168197
return nil

pkg/terraform/stages/gcp/stages.go

Lines changed: 12 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,16 @@
11
package gcp
22

33
import (
4-
"context"
54
"encoding/json"
65
"fmt"
7-
"time"
6+
"os"
87

98
igntypes "github.com/coreos/ignition/v2/config/v3_2/types"
109
"github.com/hashicorp/terraform-exec/tfexec"
1110
"github.com/pkg/errors"
1211

1312
"github.com/openshift/installer/pkg/asset"
1413
"github.com/openshift/installer/pkg/asset/ignition"
15-
gcpbootstrap "github.com/openshift/installer/pkg/asset/ignition/bootstrap/gcp"
1614
"github.com/openshift/installer/pkg/asset/lbconfig"
1715
"github.com/openshift/installer/pkg/terraform"
1816
"github.com/openshift/installer/pkg/terraform/providers"
@@ -120,21 +118,20 @@ func extractGCPLBConfig(s stages.SplitStage, directory string, terraformDir stri
120118
return "", err
121119
}
122120

123-
clusterID, ok := tfvarData["cluster_id"]
124-
if !ok {
125-
return "", fmt.Errorf("failed to read cluster id from tfvars")
126-
}
127-
128-
ctx, cancel := context.WithTimeout(context.Background(), time.Minute*1)
129-
defer cancel()
121+
// Update the ignition bootstrap variable to include the lbconfig.
122+
tfvarData["ignition_bootstrap"] = string(ignitionOutput)
130123

131-
bucketName := gcpbootstrap.GetBootstrapStorageName(clusterID.(string))
132-
bucketHandle, err := gcpbootstrap.CreateBucketHandle(ctx, bucketName)
124+
// Convert the bootstrap data and write the data back to a file. This will overwrite the original tfvars file.
125+
jsonBootstrap, err := json.Marshal(tfvarData)
133126
if err != nil {
134-
return "", fmt.Errorf("failed to create bucket handle %s: %w", bucketName, err)
127+
return "", fmt.Errorf("failed to convert bootstrap ignition to bytes: %w", err)
135128
}
136-
if err := gcpbootstrap.FillBucket(context.Background(), bucketHandle, string(ignitionOutput)); err != nil {
137-
return "", fmt.Errorf("failed to fill gcp bucket with updated boostrap ignition contents: %w", err)
129+
tfvarsFile.Data = jsonBootstrap
130+
131+
// update the value on disk to match
132+
if err := os.WriteFile(fmt.Sprintf("%s/%s", directory, tfvarsFile.Filename), jsonBootstrap, 0o600); err != nil {
133+
return "", fmt.Errorf("failed to rewrite %s: %w", tfvarsFile.Filename, err)
138134
}
135+
139136
return "", nil
140137
}

pkg/tfvars/gcp/gcp.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ type config struct {
5050
UserProvisionedDNS bool `json:"gcp_user_provisioned_dns,omitempty"`
5151
ExtraTags map[string]string `json:"gcp_extra_tags,omitempty"`
5252
IgnitionShim string `json:"gcp_ignition_shim,omitempty"`
53+
PresignedURL string `json:"gcp_signed_url"`
5354
}
5455

5556
// TFVarsSources contains the parameters to be converted into Terraform variables
@@ -66,6 +67,7 @@ type TFVarsSources struct {
6667
UserProvisionedDNS bool
6768
UserTags map[string]string
6869
IgnitionShim string
70+
PresignedURL string
6971
}
7072

7173
// TFVars generates gcp-specific Terraform variables launching the cluster.
@@ -109,6 +111,7 @@ func TFVars(sources TFVarsSources) ([]byte, error) {
109111
UserProvisionedDNS: sources.UserProvisionedDNS,
110112
ExtraTags: sources.UserTags,
111113
IgnitionShim: sources.IgnitionShim,
114+
PresignedURL: sources.PresignedURL,
112115
}
113116

114117
if masterConfig.Disks[0].EncryptionKey != nil {

0 commit comments

Comments
 (0)