clusterctl: support envsubst in clusterctl config

Signed-off-by: Stefan Büringer [email protected]

Author: sbueringer

cmd/clusterctl/client/config/cert_manager_client.go

@@ -17,8 +17,10 @@ limitations under the License.
 package config
 
 import (
+	"os"
 	"time"
 
+	"github.com/drone/envsubst/v2"
 	"github.com/pkg/errors"
 )
 
@@ -77,6 +79,12 @@ func (p *certManagerClient) Get() (CertManager, error) {
 	if userCertManager.URL != "" {
 		url = userCertManager.URL
 	}
+
+	url, err := envsubst.Eval(url, os.Getenv)
+	if err != nil {
+		return nil, errors.Wrapf(err, "unable to evaluate url: %q", url)
+	}
+
 	if userCertManager.Version != "" {
 		version = userCertManager.Version
 	}

cmd/clusterctl/client/config/cert_manager_client_test.go

@@ -17,6 +17,7 @@ limitations under the License.
 package config
 
 import (
+	"os"
 	"testing"
 
 	. "github.com/onsi/gomega"
@@ -31,6 +32,7 @@ func TestCertManagerGet(t *testing.T) {
 	tests := []struct {
 		name    string
 		fields  fields
+		envVars map[string]string
 		want    CertManager
 		wantErr bool
 	}{
@@ -50,6 +52,17 @@ func TestCertManagerGet(t *testing.T) {
 			want:    NewCertManager("foo-url", "vX.Y.Z", CertManagerDefaultTimeout.String()),
 			wantErr: false,
 		},
+		{
+			name: "return custom url with evaluated env vars if defined",
+			fields: fields{
+				reader: test.NewFakeReader().WithCertManager("${TEST_REPO_PATH}/foo-url", "vX.Y.Z", ""),
+			},
+			envVars: map[string]string{
+				"TEST_REPO_PATH": "/tmp/test",
+			},
+			want:    NewCertManager("/tmp/test/foo-url", "vX.Y.Z", CertManagerDefaultTimeout.String()),
+			wantErr: false,
+		},
 		{
 			name: "return timeout if defined",
 			fields: fields{
@@ -63,6 +76,14 @@ func TestCertManagerGet(t *testing.T) {
 		t.Run(tt.name, func(t *testing.T) {
 			g := NewWithT(t)
 
+			for k, v := range tt.envVars {
+				g.Expect(os.Setenv(k, v)).To(Succeed())
+			}
+			defer func() {
+				for k := range tt.envVars {
+					g.Expect(os.Unsetenv(k)).To(Succeed())
+				}
+			}()
 			p := &certManagerClient{
 				reader: tt.fields.reader,
 			}

cmd/clusterctl/client/config/providers_client.go

@@ -18,9 +18,11 @@ package config
 
 import (
 	"net/url"
+	"os"
 	"sort"
 	"strings"
 
+	"github.com/drone/envsubst/v2"
 	"github.com/pkg/errors"
 	"k8s.io/apimachinery/pkg/util/validation"
 
@@ -295,6 +297,12 @@ func (p *providersClient) List() ([]Provider, error) {
 	}
 
 	for _, u := range userDefinedProviders {
+		var err error
+		u.URL, err = envsubst.Eval(u.URL, os.Getenv)
+		if err != nil {
+			return nil, errors.Wrapf(err, "unable to evaluate url: %q", u.URL)
+		}
+
 		provider := NewProvider(u.Name, u.URL, u.Type)
 		if err := validateProvider(provider); err != nil {
 			return nil, errors.Wrapf(err, "error validating configuration for the %s with name %s. Please fix the providers value in clusterctl configuration file", provider.Type(), provider.Name())

cmd/clusterctl/client/config/providers_client_test.go

@@ -18,6 +18,7 @@ package config
 
 import (
 	"fmt"
+	"os"
 	"sort"
 	"testing"
 
@@ -50,6 +51,7 @@ func Test_providers_List(t *testing.T) {
 	tests := []struct {
 		name    string
 		fields  fields
+		envVars map[string]string
 		want    []Provider
 		wantErr bool
 	}{
@@ -75,6 +77,23 @@ func Test_providers_List(t *testing.T) {
 			want:    defaultsAndZZZ,
 			wantErr: false,
 		},
+		{
+			name: "Returns user defined provider configurations with evaluated env vars",
+			fields: fields{
+				configGetter: test.NewFakeReader().
+					WithVar(
+						ProvidersConfigKey,
+						"- name: \"zzz\"\n"+
+							"  url: \"${TEST_REPO_PATH}/infrastructure-components.yaml\"\n"+
+							"  type: \"InfrastructureProvider\"\n",
+					),
+			},
+			envVars: map[string]string{
+				"TEST_REPO_PATH": "https://zzz",
+			},
+			want:    defaultsAndZZZ,
+			wantErr: false,
+		},
 		{
 			name: "User defined provider configurations override defaults",
 			fields: fields{
@@ -120,6 +139,14 @@ func Test_providers_List(t *testing.T) {
 		t.Run(tt.name, func(t *testing.T) {
 			g := NewWithT(t)
 
+			for k, v := range tt.envVars {
+				g.Expect(os.Setenv(k, v)).To(Succeed())
+			}
+			defer func() {
+				for k := range tt.envVars {
+					g.Expect(os.Unsetenv(k)).To(Succeed())
+				}
+			}()
 			p := &providersClient{
 				reader: tt.fields.configGetter,
 			}

cmd/clusterctl/client/repository/overrides.go

@@ -17,14 +17,17 @@ limitations under the License.
 package repository
 
 import (
+	"fmt"
 	"os"
 	"path/filepath"
 	"strings"
 
+	"github.com/drone/envsubst/v2"
 	"github.com/pkg/errors"
 	"k8s.io/client-go/util/homedir"
 
 	"sigs.k8s.io/cluster-api/cmd/clusterctl/client/config"
+	logf "sigs.k8s.io/cluster-api/cmd/clusterctl/log"
 )
 
 const (
@@ -69,6 +72,13 @@ func (o *overrides) Path() string {
 	f, err := o.configVariablesClient.Get(overrideFolderKey)
 	if err == nil && strings.TrimSpace(f) != "" {
 		basepath = f
+
+		evaluatedBasePath, err := envsubst.Eval(basepath, os.Getenv)
+		if err != nil {
+			logf.Log.Info(fmt.Sprintf("⚠️overridesFolder %q could not be evaluated: %v", basepath, err))
+		} else {
+			basepath = evaluatedBasePath
+		}
 	}
 
 	return filepath.Join(

cmd/clusterctl/client/repository/overrides_test.go

@@ -33,6 +33,7 @@ func TestOverrides(t *testing.T) {
 	tests := []struct {
 		name            string
 		configVarClient config.VariablesClient
+		envVars         map[string]string
 		expectedPath    string
 	}{
 		{
@@ -55,11 +56,28 @@ func TestOverrides(t *testing.T) {
 			configVarClient: test.NewFakeVariableClient().WithVar(overrideFolderKey, "/Users/foobar/workspace/releases"),
 			expectedPath:    "/Users/foobar/workspace/releases/infrastructure-myinfra/v1.0.1/infra-comp.yaml",
 		},
+		{
+			name:            "uses overrides folder from the config variables with evaluated env vars",
+			configVarClient: test.NewFakeVariableClient().WithVar(overrideFolderKey, "${TEST_REPO_PATH}/releases"),
+			envVars: map[string]string{
+				"TEST_REPO_PATH": "/tmp/test",
+			},
+			expectedPath: "/tmp/test/releases/infrastructure-myinfra/v1.0.1/infra-comp.yaml",
+		},
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			g := NewWithT(t)
+
+			for k, v := range tt.envVars {
+				g.Expect(os.Setenv(k, v)).To(Succeed())
+			}
+			defer func() {
+				for k := range tt.envVars {
+					g.Expect(os.Unsetenv(k)).To(Succeed())
+				}
+			}()
 			provider := config.NewProvider("myinfra", "", clusterctlv1.InfrastructureProviderType)
 			override := newOverride(&newOverrideInput{
 				configVariablesClient: tt.configVarClient,

docs/book/src/clusterctl/configuration.md

@@ -41,6 +41,8 @@ providers:
 
 See [provider contract](provider-contract.md) for instructions about how to set up a provider repository.
 
+**Note**: It is possible to use the `${HOME}` and `${CLUSTERCTL_REPOSITORY_PATH}` environment variables in `url`.
+
 ## Variables
 
 When installing a provider `clusterctl` reads a YAML file that is published in the provider repository. While executing
@@ -73,6 +75,8 @@ cert-manager:
   url: "/Users/foo/.cluster-api/dev-repository/cert-manager/latest/cert-manager.yaml"
 ```
 
+**Note**: It is possible to use the `${HOME}` and `${CLUSTERCTL_REPOSITORY_PATH}` environment variables in `url`.
+
 Similarly, it is possible to override the default version installed by clusterctl by configuring:
 
 ```yaml
@@ -103,6 +107,7 @@ You may want to migrate to a user-managed cert-manager further down the line, af
 ```bash
 kubectl get all -A --selector=clusterctl.cluster.x-k8s.io/core=cert-manager
 ```
+
 If you want to manage and install your own cert-manager, you'll need to remove this label from all API resources.
 
 <aside class="note warning">
@@ -113,7 +118,6 @@ Cluster API has a direct dependency on cert-manager. It's possible you could enc
 
 </aside>
 
-
 ## Avoiding GitHub rate limiting
 
 Follow [this](./overview.md#avoiding-github-rate-limiting)
@@ -186,6 +190,7 @@ run,
 ```bash
 clusterctl init --infrastructure aws:v0.5.0 -v5
 ```
+
 ```bash
 ...
 Using Override="infrastructure-components.yaml" Provider="infrastructure-aws" Version="v0.5.0"
@@ -200,6 +205,8 @@ directory in the clusterctl config file as
 overridesFolder: /Users/foobar/workspace/dev-releases
 ```
 
+**Note**: It is possible to use the `${HOME}` and `${CLUSTERCTL_REPOSITORY_PATH}` environment variables in `overridesFolder`.
+
 ## Image overrides
 
 <aside class="note warning">