add 'pod-cidr' and 'service-cidr' install/restore flags (#728)

* add 'pod-cidr' and 'service-cidr' install/restore flags

* f

* add custom CIDR test

* range sizes

* skip more tests

* change test IPs

* f

* f

* +x

* f

* implement pod/service cidr check

* set calico IP detection method to 'kubernetes internal ip'

* that would be silly

* induce test failure

* wait for goldpinger properly

* undo intentional breakage, enable multinode test

* f

* improve goldpinger wait failure case

* add test combining custom CIDR and proxy

* describe goldpinger pods on failure

* wide output

* update operator to v0.36.3

* update join command response

* update pod CIDR on join

* f

* populate network in install

* admin console image override

* override service cidr

* that was not the right solution

* test with only worker nodes

* return to testing with controllers

* debug

* f

* k0s says this is a hack

* skip join config overrides

* join cidrs

* remove cidr flag

* slight refactor

* 4 nodes for default cidr test

* remove server certs on join to force regeneration

* /etc/k0s/k0s.yaml

* better /etc/k0s/k0s.yaml

* f

* cleanup and reenable tests

* airgap upgrade test

* use all additional args in airgap install

* use proper IP range for registry service

* operator fix for airgap installs

* f

* use released operator

* override configs before joining node to cluster

* cleanup unneeded comment

* backup with different cidrs

* what is this file

* correct unmarshal function

* ?

* check installation object

* ???

* ?????

* are you kidding me

* include custom CIDRs in restore command

* support arbitrary args in restore-installation-airgap.exp

* postmerge compile fixes

* f

* cfg passthrough

* remove backup store validation for testing

* fix --proxy flag

* another test

* what is being symlinked

* where is this failing

* areyoukiddingme

* reenable backup store validation

* ensure I'm creating the right cluster to begin with

* operator image override

* skip restoring installation to see if registry restores properly without it

* remove ECO image override

* update ECO again

* use released operator version

* small fixups

* reenable all the tests

* f

* minor test cleanups

* move non-proxied custom cidr test out of the proxy test file

* use released kots v1.111.0

Author: laverya

.github/workflows/pull-request.yaml

@@ -212,6 +212,7 @@ jobs:
           - TestMaterialize
           - TestLocalArtifactMirror
           - TestSingleNodeAirgapUpgrade
+          - TestSingleNodeAirgapUpgradeCustomCIDR
           - TestInstallSnapshotFromReplicatedApp
           - TestMultiNodeAirgapUpgrade
           - TestSingleNodeDisasterRecovery
@@ -224,6 +225,8 @@ jobs:
           - TestProxiedEnvironment
           - TestMultiNodeHADisasterRecovery
           - TestMultiNodeAirgapHADisasterRecovery
+          - TestCustomCIDR
+          - TestProxiedCustomCIDR
         include:
           - test: TestMultiNodeAirgapUpgrade
             runner: embedded-cluster

.github/workflows/release-dev.yaml

@@ -164,6 +164,7 @@ jobs:
           - TestMaterialize
           - TestLocalArtifactMirror
           - TestSingleNodeAirgapUpgrade
+          - TestSingleNodeAirgapUpgradeCustomCIDR
           - TestInstallSnapshotFromReplicatedApp
           - TestMultiNodeAirgapUpgrade
           - TestSingleNodeDisasterRecovery
@@ -176,6 +177,8 @@ jobs:
           - TestProxiedEnvironment
           - TestMultiNodeHADisasterRecovery
           - TestMultiNodeAirgapHADisasterRecovery
+          - TestCustomCIDR
+          - TestProxiedCustomCIDR
         include:
           - test: TestMultiNodeAirgapUpgrade
             runner: embedded-cluster

Makefile

@@ -3,10 +3,12 @@ UNAME := $(shell uname)
 ARCH := $(shell uname -m)
 APP_NAME = embedded-cluster
 ADMIN_CONSOLE_CHART_REPO_OVERRIDE =
-ADMIN_CONSOLE_CHART_VERSION = 1.109.13
+ADMIN_CONSOLE_CHART_VERSION = 1.111.0
 ADMIN_CONSOLE_IMAGE_OVERRIDE =
 ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE =
-EMBEDDED_OPERATOR_CHART_VERSION = 0.36.2
+EMBEDDED_OPERATOR_CHART_URL = oci://registry.replicated.com/library
+EMBEDDED_OPERATOR_CHART_NAME = embedded-cluster-operator
+EMBEDDED_OPERATOR_CHART_VERSION = 0.36.5
 EMBEDDED_OPERATOR_UTILS_IMAGE = busybox:1.36.1
 EMBEDDED_CLUSTER_OPERATOR_IMAGE_OVERRIDE =
 OPENEBS_CHART_VERSION = 4.1.0

cmd/embedded-cluster/install.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	k0sconfig "github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1"
+	ecv1beta1 "github.com/replicatedhq/embedded-cluster-kinds/apis/v1beta1"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
 	k8syaml "sigs.k8s.io/yaml"
@@ -313,15 +314,21 @@ func materializeFiles(c *cli.Context) error {
 // createK0sConfig creates a new k0s.yaml configuration file. The file is saved in the
 // global location (as returned by defaults.PathToK0sConfig()). If a file already sits
 // there, this function returns an error.
-func ensureK0sConfig(c *cli.Context) error {
+func ensureK0sConfig(c *cli.Context) (*k0sconfig.ClusterConfig, error) {
 	cfgpath := defaults.PathToK0sConfig()
 	if _, err := os.Stat(cfgpath); err == nil {
-		return fmt.Errorf("configuration file already exists")
+		return nil, fmt.Errorf("configuration file already exists")
 	}
 	if err := os.MkdirAll(filepath.Dir(cfgpath), 0755); err != nil {
-		return fmt.Errorf("unable to create directory: %w", err)
+		return nil, fmt.Errorf("unable to create directory: %w", err)
 	}
 	cfg := config.RenderK0sConfig()
+	if c.String("pod-cidr") != "" {
+		cfg.Spec.Network.PodCIDR = c.String("pod-cidr")
+	}
+	if c.String("service-cidr") != "" {
+		cfg.Spec.Network.ServiceCIDR = c.String("service-cidr")
+	}
 	opts := []addons.Option{}
 	if c.Bool("no-prompt") {
 		opts = append(opts, addons.WithoutPrompt())
@@ -333,17 +340,21 @@ func ensureK0sConfig(c *cli.Context) error {
 		opts = append(opts, addons.WithAirgapBundle(ab))
 	}
 	if c.Bool("proxy") {
-		opts = append(opts, addons.WithProxyFromEnv())
+		opts = append(opts, addons.WithProxyFromEnv(cfg.Spec.Network.PodCIDR, cfg.Spec.Network.ServiceCIDR))
 	}
 	if c.String("http-proxy") != "" || c.String("https-proxy") != "" || c.String("no-proxy") != "" {
-		opts = append(opts, addons.WithProxyFromArgs(c.String("http-proxy"), c.String("https-proxy"), c.String("no-proxy")))
+		opts = append(opts, addons.WithProxyFromArgs(c.String("http-proxy"), c.String("https-proxy"), c.String("no-proxy"), cfg.Spec.Network.PodCIDR, cfg.Spec.Network.ServiceCIDR))
 	}
+	opts = append(opts, addons.WithNetwork(&ecv1beta1.NetworkSpec{
+		PodCIDR:     cfg.Spec.Network.PodCIDR,
+		ServiceCIDR: cfg.Spec.Network.ServiceCIDR,
+	}))
 	if err := config.UpdateHelmConfigs(cfg, opts...); err != nil {
-		return fmt.Errorf("unable to update helm configs: %w", err)
+		return nil, fmt.Errorf("unable to update helm configs: %w", err)
 	}
 	var err error
 	if cfg, err = applyUnsupportedOverrides(c, cfg); err != nil {
-		return fmt.Errorf("unable to apply unsupported overrides: %w", err)
+		return nil, fmt.Errorf("unable to apply unsupported overrides: %w", err)
 	}
 	if c.String("airgap-bundle") != "" {
 		// update the k0s config to install with airgap
@@ -352,17 +363,18 @@ func ensureK0sConfig(c *cli.Context) error {
 	}
 	data, err := k8syaml.Marshal(cfg)
 	if err != nil {
-		return fmt.Errorf("unable to marshal config: %w", err)
+		return nil, fmt.Errorf("unable to marshal config: %w", err)
 	}
 	fp, err := os.OpenFile(cfgpath, os.O_RDWR|os.O_CREATE, 0600)
 	if err != nil {
-		return fmt.Errorf("unable to create config file: %w", err)
+		return nil, fmt.Errorf("unable to create config file: %w", err)
 	}
 	defer fp.Close()
 	if _, err := fp.Write(data); err != nil {
-		return fmt.Errorf("unable to write config file: %w", err)
+		return nil, fmt.Errorf("unable to write config file: %w", err)
 	}
-	return nil
+
+	return cfg, nil
 }
 
 // applyUnsupportedOverrides applies overrides to the k0s configuration. Applies first the
@@ -438,19 +450,20 @@ func waitForK0s() error {
 }
 
 // installAndWaitForK0s installs the k0s binary and waits for it to be ready
-func installAndWaitForK0s(c *cli.Context) error {
+func installAndWaitForK0s(c *cli.Context) (*k0sconfig.ClusterConfig, error) {
 	loading := spinner.Start()
 	defer loading.Close()
 	loading.Infof("Installing %s node", defaults.BinaryName())
 	logrus.Debugf("creating k0s configuration file")
-	if err := ensureK0sConfig(c); err != nil {
+	cfg, err := ensureK0sConfig(c)
+	if err != nil {
 		err := fmt.Errorf("unable to create config file: %w", err)
 		metrics.ReportApplyFinished(c, err)
-		return err
+		return nil, err
 	}
-	var proxy *Proxy
+	var proxy *ecv1beta1.ProxySpec
 	if c.String("http-proxy") != "" || c.String("https-proxy") != "" || c.String("no-proxy") != "" {
-		proxy = &Proxy{
+		proxy = &ecv1beta1.ProxySpec{
 			HTTPProxy:  c.String("http-proxy"),
 			HTTPSProxy: c.String("https-proxy"),
 			NoProxy:    strings.Join(append(defaults.DefaultNoProxy, c.String("no-proxy")), ","),
@@ -460,28 +473,28 @@ func installAndWaitForK0s(c *cli.Context) error {
 	if err := createSystemdUnitFiles(false, proxy); err != nil {
 		err := fmt.Errorf("unable to create systemd unit files: %w", err)
 		metrics.ReportApplyFinished(c, err)
-		return err
+		return nil, err
 	}
 
 	logrus.Debugf("installing k0s")
 	if err := installK0s(); err != nil {
 		err := fmt.Errorf("unable update cluster: %w", err)
 		metrics.ReportApplyFinished(c, err)
-		return err
+		return nil, err
 	}
 	loading.Infof("Waiting for %s node to be ready", defaults.BinaryName())
 	logrus.Debugf("waiting for k0s to be ready")
 	if err := waitForK0s(); err != nil {
 		err := fmt.Errorf("unable to wait for node: %w", err)
 		metrics.ReportApplyFinished(c, err)
-		return err
+		return nil, err
 	}
 	loading.Infof("Node installation finished!")
-	return nil
+	return cfg, nil
 }
 
 // runOutro calls Outro() in all enabled addons by means of Applier.
-func runOutro(c *cli.Context, adminConsolePwd string) error {
+func runOutro(c *cli.Context, cfg *k0sconfig.ClusterConfig, adminConsolePwd string) error {
 	os.Setenv("KUBECONFIG", defaults.PathToKubeConfig())
 	opts := []addons.Option{}
 
@@ -506,8 +519,12 @@ func runOutro(c *cli.Context, adminConsolePwd string) error {
 	}
 	opts = append(opts, addons.WithAdminConsolePassword(adminConsolePwd))
 	if c.String("http-proxy") != "" || c.String("https-proxy") != "" || c.String("no-proxy") != "" {
-		opts = append(opts, addons.WithProxyFromArgs(c.String("http-proxy"), c.String("https-proxy"), c.String("no-proxy")))
+		opts = append(opts, addons.WithProxyFromArgs(c.String("http-proxy"), c.String("https-proxy"), c.String("no-proxy"), cfg.Spec.Network.PodCIDR, cfg.Spec.Network.ServiceCIDR))
 	}
+	opts = append(opts, addons.WithNetwork(&ecv1beta1.NetworkSpec{
+		PodCIDR:     cfg.Spec.Network.PodCIDR,
+		ServiceCIDR: cfg.Spec.Network.ServiceCIDR,
+	}))
 	return addons.NewApplier(opts...).Outro(c.Context)
 }
 
@@ -588,6 +605,16 @@ var installCommand = &cli.Command{
 			Usage:  "Use the system proxy settings for the install operation. These variables are currently only passed through to Velero and the Admin Console.",
 			Hidden: true,
 		},
+		&cli.StringFlag{
+			Name:   "pod-cidr",
+			Usage:  "pod CIDR range to use for the installation",
+			Hidden: false,
+		},
+		&cli.StringFlag{
+			Name:   "service-cidr",
+			Usage:  "service CIDR range to use for the installation",
+			Hidden: false,
+		},
 		&cli.BoolFlag{
 			Name:  "skip-host-preflights",
 			Usage: "Skip host preflight checks. This is not recommended unless you are sure your system is compatible.",
@@ -638,11 +665,12 @@ var installCommand = &cli.Command{
 			metrics.ReportApplyFinished(c, err)
 			return err
 		}
-		if err := installAndWaitForK0s(c); err != nil {
+		cfg, err := installAndWaitForK0s(c)
+		if err != nil {
 			return err
 		}
 		logrus.Debugf("running outro")
-		if err := runOutro(c, adminConsolePwd); err != nil {
+		if err := runOutro(c, cfg, adminConsolePwd); err != nil {
 			metrics.ReportApplyFinished(c, err)
 			return err
 		}

cmd/embedded-cluster/join.go

@@ -13,6 +13,7 @@ import (
 	"github.com/google/uuid"
 	"github.com/k0sproject/dig"
 	k0sconfig "github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1"
+	ecv1beta1 "github.com/replicatedhq/embedded-cluster-kinds/apis/v1beta1"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
 	"gopkg.in/yaml.v2"
@@ -33,22 +34,17 @@ import (
 	"github.com/replicatedhq/embedded-cluster/pkg/spinner"
 )
 
-type Proxy struct {
-	HTTPProxy  string `json:"httpProxy"`
-	HTTPSProxy string `json:"httpsProxy"`
-	NoProxy    string `json:"noProxy"`
-}
-
 // JoinCommandResponse is the response from the kots api we use to fetch the k0s join token.
 type JoinCommandResponse struct {
-	K0sJoinCommand            string    `json:"k0sJoinCommand"`
-	K0sToken                  string    `json:"k0sToken"`
-	ClusterID                 uuid.UUID `json:"clusterID"`
-	K0sUnsupportedOverrides   string    `json:"k0sUnsupportedOverrides"`
-	EndUserK0sConfigOverrides string    `json:"endUserK0sConfigOverrides"`
-	MetricsBaseURL            string    `json:"metricsBaseURL"`
-	AirgapRegistryAddress     string    `json:"airgapRegistryAddress"`
-	Proxy                     *Proxy    `json:"proxy"`
+	K0sJoinCommand            string                 `json:"k0sJoinCommand"`
+	K0sToken                  string                 `json:"k0sToken"`
+	ClusterID                 uuid.UUID              `json:"clusterID"`
+	K0sUnsupportedOverrides   string                 `json:"k0sUnsupportedOverrides"`
+	EndUserK0sConfigOverrides string                 `json:"endUserK0sConfigOverrides"`
+	MetricsBaseURL            string                 `json:"metricsBaseURL"`
+	AirgapRegistryAddress     string                 `json:"airgapRegistryAddress"`
+	Proxy                     *ecv1beta1.ProxySpec   `json:"proxy"`
+	Network                   *ecv1beta1.NetworkSpec `json:"network"`
 }
 
 // extractK0sConfigOverridePatch parses the provided override and returns a dig.Mapping that
@@ -241,11 +237,10 @@ var joinCommand = &cli.Command{
 			return err
 		}
 
-		logrus.Debugf("joining node to cluster")
-		if err := runK0sInstallCommand(jcmd.K0sJoinCommand); err != nil {
-			err := fmt.Errorf("unable to join node to cluster: %w", err)
+		logrus.Debugf("overriding network configuration")
+		if err := applyNetworkConfiguration(jcmd); err != nil {
+			err := fmt.Errorf("unable to apply network configuration: %w", err)
 			metrics.ReportJoinFailed(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID, err)
-			return err
 		}
 
 		logrus.Debugf("applying configuration overrides")
@@ -255,6 +250,13 @@ var joinCommand = &cli.Command{
 			return err
 		}
 
+		logrus.Debugf("joining node to cluster")
+		if err := runK0sInstallCommand(jcmd.K0sJoinCommand); err != nil {
+			err := fmt.Errorf("unable to join node to cluster: %w", err)
+			metrics.ReportJoinFailed(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID, err)
+			return err
+		}
+
 		if err := startAndWaitForK0s(c, jcmd); err != nil {
 			return err
 		}
@@ -297,6 +299,24 @@ var joinCommand = &cli.Command{
 	},
 }
 
+func applyNetworkConfiguration(jcmd *JoinCommandResponse) error {
+	if jcmd.Network != nil {
+		clusterSpec := k0sconfig.DefaultClusterConfig()
+		clusterSpec.Spec.Network.PodCIDR = jcmd.Network.PodCIDR
+		clusterSpec.Spec.Network.ServiceCIDR = jcmd.Network.ServiceCIDR
+		clusterSpecYaml, err := k8syaml.Marshal(clusterSpec)
+
+		if err != nil {
+			return fmt.Errorf("unable to marshal cluster spec: %w", err)
+		}
+		err = os.WriteFile(defaults.PathToK0sConfig(), clusterSpecYaml, 0644)
+		if err != nil {
+			return fmt.Errorf("unable to write cluster spec to /etc/k0s/k0s.yaml: %w", err)
+		}
+	}
+	return nil
+}
+
 // applyJoinConfigurationOverrides applies both config overrides received from the kots api.
 // Applies first the EmbeddedOverrides and then the EndUserOverrides.
 func applyJoinConfigurationOverrides(jcmd *JoinCommandResponse) error {
@@ -419,6 +439,7 @@ func runK0sInstallCommand(fullcmd string) error {
 	if strings.Contains(fullcmd, "controller") {
 		args = append(args, "--disable-components", "konnectivity-server", "--enable-dynamic-config")
 	}
+
 	if _, err := helpers.RunCommand(args[0], args[1:]...); err != nil {
 		return err
 	}