[release/v1.45] improve kubeconfig/gcp handling of credentials in e2e tests (#1404) (#1409)

Author: xrstf

docs/cloud-provider.md

@@ -147,7 +147,8 @@ tags:
 ### machine.spec.providerConfig.cloudProviderSpec
 
 ```yaml
-serviceAccount: "<< GOOGLE_SERVICE_ACCOUNT >>"
+# The service account needs to be base64-encoded.
+serviceAccount: "<< GOOGLE_SERVICE_ACCOUNT_BASE64 >>"
 # See https://cloud.google.com/compute/docs/regions-zones/
 zone: "europe-west3-a"
 # See https://cloud.google.com/compute/docs/machine-types
@@ -295,8 +296,8 @@ tags:
 
 ### machine.spec.providerConfig.cloudProviderSpec
 ```yaml
-# kubeconfig to access KubeVirt cluster
-kubeconfig: '<< KUBECONFIG >>'
+# base64-encoded kubeconfig to access KubeVirt cluster
+kubeconfig: '<< KUBECONFIG_BASE64 >>'
 # KubeVirt namespace
 namespace: kube-system
 # kubernetes storage class

examples/gce-machinedeployment.yaml

@@ -6,8 +6,13 @@ metadata:
   name: machine-controller-gce
   namespace: kube-system
 type: Opaque
-stringData:
-  serviceAccount: "<< GOOGLE_SERVICE_ACCOUNT >>"
+data:
+  # The base64 encoding here is only to satisfy Kubernetes'
+  # Secret storage and to prevent multiline string replacement
+  # issues if we used stringData here (because the GCP SA is
+  # a multiline JSON string).
+  serviceAccount: "<< GOOGLE_SERVICE_ACCOUNT_BASE64 >>"
+
 ---
 apiVersion: "cluster.k8s.io/v1alpha1"
 kind: MachineDeployment

examples/kubevirt-machinedeployment.yaml

@@ -32,7 +32,9 @@ spec:
             cpus: "1"
             memory: "2048M"
             kubeconfig:
-              value: '<< KUBECONFIG >>'
+              # Can also be set via the env var 'KUBEVIRT_KUBECONFIG' on the machine-controller.
+              # If instead specified directly, this value should be a base64 encoded kubeconfig.
+              value: "<< KUBECONFIG_BASE64 >>"
             namespace: kube-system
           # Can also be `centos`, must align with he configured registryImage above
           operatingSystem: "ubuntu"

pkg/cloudprovider/provider/gce/config.go

@@ -198,19 +198,25 @@ func newConfig(resolver *providerconfig.ConfigVarResolver, spec v1alpha1.Provide
 // postprocessServiceAccount processes the service account and creates a JWT configuration
 // out of it.
 func (cfg *config) postprocessServiceAccount() error {
-	sa, err := base64.StdEncoding.DecodeString(cfg.serviceAccount)
-	if err != nil {
-		return fmt.Errorf("failed to decode base64 service account: %v", err)
+	sa := cfg.serviceAccount
+
+	// safely decode the service account, in case we did not read the value
+	// from a "known-safe" location (like the MachineDeployment), but from
+	// an environment variable.
+	decoded, err := base64.StdEncoding.DecodeString(cfg.serviceAccount)
+	if err == nil {
+		sa = string(decoded)
 	}
+
 	sam := map[string]string{}
-	err = json.Unmarshal(sa, &sam)
+	err = json.Unmarshal([]byte(sa), &sam)
 	if err != nil {
-		return fmt.Errorf("failed unmarshalling service account: %v", err)
+		return fmt.Errorf("failed unmarshalling service account: %w", err)
 	}
 	cfg.projectID = sam["project_id"]
-	cfg.jwtConfig, err = google.JWTConfigFromJSON(sa, compute.ComputeScope)
+	cfg.jwtConfig, err = google.JWTConfigFromJSON([]byte(sa), compute.ComputeScope)
 	if err != nil {
-		return fmt.Errorf("failed preparing JWT: %v", err)
+		return fmt.Errorf("failed preparing JWT: %w", err)
 	}
 	return nil
 }

pkg/cloudprovider/provider/gce/types/types.go

@@ -30,6 +30,7 @@ import (
 // CloudProviderSpec contains the specification of the cloud provider taken
 // from the provider configuration.
 type CloudProviderSpec struct {
+	// ServiceAccount must be base64-encoded.
 	ServiceAccount        providerconfigtypes.ConfigVarString `json:"serviceAccount,omitempty"`
 	Zone                  providerconfigtypes.ConfigVarString `json:"zone"`
 	MachineType           providerconfigtypes.ConfigVarString `json:"machineType"`

test/e2e/provisioning/all_e2e_test.go

@@ -20,6 +20,7 @@ package provisioning
 
 import (
 	"context"
+	"encoding/base64"
 	"flag"
 	"fmt"
 	"os"
@@ -290,12 +291,23 @@ func TestKubevirtProvisioningE2E(t *testing.T) {
 	selector := OsSelector("ubuntu", "centos", "flatcar")
 
 	params := []string{
-		fmt.Sprintf("<< KUBECONFIG >>=%s", kubevirtKubeconfig),
+		fmt.Sprintf("<< KUBECONFIG_BASE64 >>=%s", safeBase64Encoding(kubevirtKubeconfig)),
 	}
 
 	runScenarios(t, selector, params, kubevirtManifest, fmt.Sprintf("kubevirt-%s", *testRunIdentifier))
 }
 
+// safeBase64Encoding takes a value and encodes it with base64
+// if it is not already encoded.
+func safeBase64Encoding(value string) string {
+	// If there was no error, the original value was already encoded.
+	if _, err := base64.StdEncoding.DecodeString(value); err == nil {
+		return value
+	}
+
+	return base64.StdEncoding.EncodeToString([]byte(value))
+}
+
 func TestOpenstackProvisioningE2E(t *testing.T) {
 	t.Parallel()
 
@@ -690,8 +702,9 @@ func TestGCEProvisioningE2E(t *testing.T) {
 	// Act. GCE does not support CentOS.
 	selector := OsSelector("ubuntu")
 	params := []string{
-		fmt.Sprintf("<< GOOGLE_SERVICE_ACCOUNT >>=%s", googleServiceAccount),
+		fmt.Sprintf("<< GOOGLE_SERVICE_ACCOUNT_BASE64 >>=%s", safeBase64Encoding(googleServiceAccount)),
 	}
+
 	runScenarios(t, selector, params, GCEManifest, fmt.Sprintf("gce-%s", *testRunIdentifier))
 }
 

test/e2e/provisioning/testdata/machinedeployment-gce.yaml

@@ -24,8 +24,9 @@ spec:
             - "<< YOUR_PUBLIC_KEY >>"
           cloudProvider: "gce"
           cloudProviderSpec:
-            # If empty, can be set via GOOGLE_SERVICE_ACCOUNT env var
-            serviceAccount: "<< GOOGLE_SERVICE_ACCOUNT >>"
+            # If empty, can be set via GOOGLE_SERVICE_ACCOUNT env var. The environment variable
+            # should be plaintext. The value in the cloudProviderSpec however must be base64-encoded.
+            serviceAccount: "<< GOOGLE_SERVICE_ACCOUNT_BASE64 >>"
             # See https://cloud.google.com/compute/docs/regions-zones/
             zone: "europe-west3-a"
             # See https://cloud.google.com/compute/docs/machine-types

test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml

@@ -36,7 +36,7 @@ spec:
               nameservers:
                 - 8.8.8.8
             kubeconfig:
-              value: '<< KUBECONFIG >>'
+              value: '<< KUBECONFIG_BASE64 >>'
             namespace: kube-system
           operatingSystem: "<< OS_NAME >>"
           operatingSystemSpec: