Validate custom cidrs for airgap / proxy (#3142)

* Validate custom cidrs for airgap / proxy

* validate no proxy in charts as well

Author: sgalsaleh

.github/workflows/ci.yaml

@@ -904,7 +904,6 @@ jobs:
           - TestInstallFromReplicatedApp
           - TestUpgradeFromReplicatedAppPreviousK0s
           - TestResetAndReinstall
-          - TestInstallSnapshotFromReplicatedApp
           - TestMultiNodeInstallation
           - TestMultiNodeHAInstallation
           - TestSingleNodeDisasterRecovery
@@ -985,7 +984,6 @@ jobs:
           - TestSingleNodeAirgapUpgrade
           - TestSingleNodeAirgapUpgradeSelinux
           - TestSingleNodeAirgapUpgradeConfigValues
-          - TestSingleNodeAirgapUpgradeCustomCIDR
           - TestMultiNodeAirgapUpgrade
           - TestMultiNodeAirgapUpgradeSameK0s
           - TestMultiNodeAirgapUpgradePreviousStable

.github/workflows/release-prod.yaml

@@ -520,7 +520,6 @@ jobs:
           - TestInstallFromReplicatedApp
           - TestUpgradeFromReplicatedApp
           - TestResetAndReinstall
-          - TestInstallSnapshotFromReplicatedApp
           - TestMultiNodeInstallation
           - TestMultiNodeHAInstallation
           - TestSingleNodeDisasterRecovery
@@ -600,7 +599,6 @@ jobs:
           - TestSingleNodeAirgapUpgrade
           - TestSingleNodeAirgapUpgradeSelinux
           - TestSingleNodeAirgapUpgradeConfigValues
-          - TestSingleNodeAirgapUpgradeCustomCIDR
           - TestMultiNodeAirgapUpgrade
           - TestMultiNodeAirgapUpgradeSameK0s
           - TestMultiNodeAirgapUpgradePreviousStable

e2e/install_test.go

@@ -13,7 +13,6 @@ import (
 
 	"github.com/replicatedhq/embedded-cluster/e2e/cluster/cmx"
 	"github.com/replicatedhq/embedded-cluster/e2e/cluster/docker"
-	"github.com/replicatedhq/embedded-cluster/e2e/cluster/lxd"
 )
 
 func TestSingleNodeInstallation(t *testing.T) {
@@ -851,302 +850,6 @@ func TestSingleNodeAirgapUpgradeSelinux(t *testing.T) {
 	t.Logf("%s: test complete", time.Now().Format(time.RFC3339))
 }
 
-func TestSingleNodeAirgapUpgradeCustomCIDR(t *testing.T) {
-	t.Parallel()
-
-	RequireEnvVars(t, []string{"SHORT_SHA"})
-
-	tc := cmx.NewCluster(&cmx.ClusterInput{
-		T:            t,
-		Nodes:        1,
-		Distribution: "ubuntu",
-		Version:      "22.04",
-	})
-	defer tc.Cleanup()
-
-	t.Logf("%s: downloading airgap files on node 0", time.Now().Format(time.RFC3339))
-	// Previous stable EC version with a -1 minor k0s version
-	initialVersion := fmt.Sprintf("appver-%s-previous-stable", os.Getenv("SHORT_SHA"))
-	runInParallel(t,
-		func(t *testing.T) error {
-			return downloadAirgapBundleOnNode(t, tc, 0, initialVersion, AirgapInstallBundlePath, AirgapLicenseID)
-		}, func(t *testing.T) error {
-			return downloadAirgapBundleOnNode(t, tc, 0, fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA")), AirgapUpgradeBundlePath, AirgapLicenseID)
-		},
-	)
-
-	t.Logf("%s: airgapping cluster", time.Now().Format(time.RFC3339))
-	if err := tc.Airgap(); err != nil {
-		t.Fatalf("failed to airgap cluster: %v", err)
-	}
-
-	t.Logf("%s: preparing embedded cluster airgap files", time.Now().Format(time.RFC3339))
-	line := []string{"airgap-prepare.sh"}
-	if _, _, err := tc.RunCommandOnNode(0, line); err != nil {
-		t.Fatalf("fail to prepare airgap files on node %s: %v", tc.Nodes[0], err)
-	}
-
-	installSingleNodeWithOptions(t, tc, installOptions{
-		isAirgap: true,
-		version:  initialVersion,
-		cidr:     "172.16.0.0/15",
-	})
-
-	if stdout, stderr, err := tc.SetupPlaywrightAndRunTest("deploy-app"); err != nil {
-		t.Fatalf("fail to run playwright test deploy-app: %v: %s: %s", err, stdout, stderr)
-	}
-
-	t.Logf("%s: checking installation state after app deployment", time.Now().Format(time.RFC3339))
-	line = []string{"check-airgap-installation-state.sh", initialVersion, k8sVersionPreviousStable()}
-	if _, _, err := tc.RunCommandOnNode(0, line); err != nil {
-		t.Fatalf("fail to check installation state: %v", err)
-	}
-
-	t.Logf("%s: running airgap update", time.Now().Format(time.RFC3339))
-	line = []string{"airgap-update.sh"}
-	if _, _, err := tc.RunCommandOnNode(0, line); err != nil {
-		t.Fatalf("fail to run airgap update: %v", err)
-	}
-
-	appUpgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA"))
-	testArgs := []string{appUpgradeVersion}
-
-	t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339))
-	if stdout, stderr, err := tc.RunPlaywrightTest("deploy-upgrade", testArgs...); err != nil {
-		t.Fatalf("fail to run playwright test deploy-upgrade: %v: %s: %s", err, stdout, stderr)
-	}
-
-	checkPostUpgradeState(t, tc)
-
-	// ensure that the cluster is using the right IP ranges.
-	t.Logf("%s: checking service and pod IP addresses", time.Now().Format(time.RFC3339))
-
-	// we have used --cidr 172.16.0.0/15 during install time so pods are
-	// expected to be in the 172.16.0.0/16 range while services are in the
-	// 172.17.0.0/16 range.
-	podregex := `172\\.16\\.[0-9]\\+\\.[0-9]\\+`
-	svcregex := `172\\.17\\.[0-9]\\+\\.[0-9]\\+`
-
-	if stdout, stderr, err := tc.RunCommandOnNode(0, []string{"check-cidr-ranges.sh", podregex, svcregex}); err != nil {
-		t.Log(stdout)
-		t.Log(stderr)
-		t.Fatalf("fail to check addresses on node %s: %v", tc.Nodes[0], err)
-	}
-
-	t.Logf("%s: test complete", time.Now().Format(time.RFC3339))
-}
-
-func TestAirgapUpgradeFromEC18(t *testing.T) {
-	t.Parallel()
-
-	RequireEnvVars(t, []string{"SHORT_SHA"})
-
-	withEnv := map[string]string{"KUBECONFIG": "/var/lib/k0s/pki/admin.conf"}
-
-	appVer := fmt.Sprintf("appver-%s-1.8.0-k8s-1.28", os.Getenv("SHORT_SHA"))
-
-	t.Logf("%s: downloading airgap files", time.Now().Format(time.RFC3339))
-	airgapInstallBundlePath := "/tmp/airgap-install-bundle.tar.gz"
-	airgapUpgradeBundlePath := "/tmp/airgap-upgrade-bundle.tar.gz"
-	airgapUpgrade2BundlePath := "/tmp/airgap-upgrade2-bundle.tar.gz"
-	runInParallel(t,
-		func(t *testing.T) error {
-			return downloadAirgapBundle(t, appVer, airgapInstallBundlePath, AirgapLicenseID)
-		}, func(t *testing.T) error {
-			return downloadAirgapBundle(t, fmt.Sprintf("appver-%s-noop", os.Getenv("SHORT_SHA")), airgapUpgradeBundlePath, AirgapLicenseID)
-		}, func(t *testing.T) error {
-			return downloadAirgapBundle(t, fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA")), airgapUpgrade2BundlePath, AirgapLicenseID)
-		},
-	)
-
-	tc := lxd.NewCluster(&lxd.ClusterInput{
-		T:                        t,
-		Nodes:                    2,
-		Image:                    "debian/12",
-		WithProxy:                true,
-		AirgapInstallBundlePath:  airgapInstallBundlePath,
-		AirgapUpgradeBundlePath:  airgapUpgradeBundlePath,
-		AirgapUpgrade2BundlePath: airgapUpgrade2BundlePath,
-		LowercaseNodeNames:       true,
-	})
-	defer tc.Cleanup(withEnv)
-
-	// delete airgap bundles once they've been copied to the nodes
-	if err := os.Remove(airgapInstallBundlePath); err != nil {
-		t.Logf("failed to remove airgap install bundle: %v", err)
-	}
-	if err := os.Remove(airgapUpgradeBundlePath); err != nil {
-		t.Logf("failed to remove airgap upgrade bundle: %v", err)
-	}
-	if err := os.Remove(airgapUpgrade2BundlePath); err != nil {
-		t.Logf("failed to remove airgap upgrade bundle: %v", err)
-	}
-
-	// upgrade airgap bundle is only needed on the first node
-	line := []string{"rm", "/assets/ec-release-upgrade.tgz"}
-	if _, _, err := tc.RunCommandOnNode(1, line); err != nil {
-		t.Fatalf("fail to remove upgrade airgap bundle on node %s: %v", tc.Nodes[1], err)
-	}
-
-	// install "curl" dependency on node 0 for app version checks.
-	tc.InstallTestDependenciesDebian(t, 0, true)
-
-	t.Logf("%s: preparing embedded cluster airgap files", time.Now().Format(time.RFC3339))
-	line = []string{"airgap-prepare.sh"}
-	if _, _, err := tc.RunCommandOnNode(0, line); err != nil {
-		t.Fatalf("fail to prepare airgap files on node %s: %v", tc.Nodes[0], err)
-	}
-
-	installSingleNodeWithOptions(t, tc, installOptions{
-		isAirgap: true,
-		version:  appVer,
-		withEnv:  withEnv,
-	})
-	// remove the airgap bundle after installation
-	line = []string{"rm", "/assets/release.airgap"}
-	if _, _, err := tc.RunCommandOnNode(0, line); err != nil {
-		t.Fatalf("fail to remove airgap bundle on node %s: %v", tc.Nodes[0], err)
-	}
-
-	if err := tc.SetupPlaywright(withEnv); err != nil {
-		t.Fatalf("fail to setup playwright: %v", err)
-	}
-	if stdout, stderr, err := tc.RunPlaywrightTest("deploy-ec18-app"); err != nil {
-		t.Fatalf("fail to run playwright test deploy-ec18-app: %v: %s: %s", err, stdout, stderr)
-	}
-
-	// generate worker node join command.
-	t.Logf("%s: generating a new worker token command", time.Now().Format(time.RFC3339))
-	stdout, stderr, err := tc.RunPlaywrightTest("get-ec18-join-worker-command")
-	if err != nil {
-		t.Fatalf("fail to generate worker join token:\nstdout: %s\nstderr: %s", stdout, stderr)
-	}
-	workerCommand, err := findJoinCommandInOutput(stdout)
-	if err != nil {
-		t.Fatalf("fail to find the join command in the output: %v", err)
-	}
-	t.Log("worker join token command:", workerCommand)
-
-	// join the worker node
-	t.Logf("%s: preparing embedded cluster airgap files on worker node", time.Now().Format(time.RFC3339))
-	line = []string{"airgap-prepare.sh"}
-	if _, _, err := tc.RunCommandOnNode(1, line); err != nil {
-		t.Fatalf("fail to prepare airgap files on worker node: %v", err)
-	}
-	t.Logf("%s: joining worker node to the cluster", time.Now().Format(time.RFC3339))
-	if _, _, err := tc.RunCommandOnNode(1, strings.Split(workerCommand, " ")); err != nil {
-		t.Fatalf("fail to join worker node to the cluster: %v", err)
-	}
-	// remove artifacts after joining to save space
-	line = []string{"rm", "/assets/release.airgap"}
-	if _, _, err := tc.RunCommandOnNode(1, line); err != nil {
-		t.Fatalf("fail to remove airgap bundle on worker node: %v", err)
-	}
-	line = []string{"rm", "/usr/local/bin/embedded-cluster"}
-	if _, _, err := tc.RunCommandOnNode(1, line); err != nil {
-		t.Fatalf("fail to remove embedded-cluster binary on worker node: %v", err)
-	}
-	line = []string{"rm", "/var/lib/embedded-cluster/bin/embedded-cluster"}
-	if _, _, err := tc.RunCommandOnNode(1, line); err != nil {
-		t.Fatalf("fail to remove embedded-cluster binary on node %s: %v", tc.Nodes[0], err)
-	}
-
-	// wait for the nodes to report as ready.
-	t.Logf("%s: all nodes joined, waiting for them to be ready", time.Now().Format(time.RFC3339))
-	stdout, _, err = tc.RunCommandOnNode(0, []string{"wait-for-ready-nodes.sh", "2"}, withEnv)
-	if err != nil {
-		t.Log(stdout)
-		t.Fatalf("fail to wait for ready nodes: %v", err)
-	}
-
-	t.Logf("%s: checking installation state after app deployment", time.Now().Format(time.RFC3339))
-	line = []string{
-		"check-airgap-installation-state.sh",
-		// the initially installed version is 1.8.0+k8s-1.28
-		// the '+' character is problematic in the regex used to validate the version, so we use '.' instead
-		appVer,
-		"v1.28.11",
-	}
-	if _, _, err := tc.RunCommandOnNode(0, line, withEnv); err != nil {
-		t.Fatalf("fail to check installation state: %v", err)
-	}
-
-	t.Logf("%s: running airgap update", time.Now().Format(time.RFC3339))
-	line = []string{"airgap-update.sh"}
-	if _, _, err := tc.RunCommandOnNode(0, line); err != nil {
-		t.Fatalf("fail to run airgap update: %v", err)
-	}
-	// remove the airgap bundle after upgrade
-	line = []string{"rm", "/assets/upgrade/release.airgap"}
-	if _, _, err := tc.RunCommandOnNode(0, line); err != nil {
-		t.Fatalf("fail to remove airgap bundle on node %s: %v", tc.Nodes[0], err)
-	}
-
-	appUpgradeVersion := fmt.Sprintf("appver-%s-noop", os.Getenv("SHORT_SHA"))
-	testArgs := []string{appUpgradeVersion}
-
-	t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339))
-	if stdout, stderr, err := tc.RunPlaywrightTest("deploy-upgrade", testArgs...); err != nil {
-		t.Fatalf("fail to run playwright test deploy-upgrade: %v: %s: %s", err, stdout, stderr)
-	}
-
-	t.Logf("%s: checking installation state after noop upgrade", time.Now().Format(time.RFC3339))
-	line = []string{"check-airgap-installation-state.sh", appUpgradeVersion, k8sVersion()}
-	if stdout, stderr, err := tc.RunCommandOnNode(0, line, withEnv); err != nil {
-		t.Fatalf("fail to check installation state: %v: %s: %s", err, stdout, stderr)
-	}
-
-	t.Logf("%s: running second airgap update", time.Now().Format(time.RFC3339))
-	line = []string{"airgap-update2.sh"}
-	if _, _, err := tc.RunCommandOnNode(0, line, withEnv); err != nil {
-		t.Fatalf("fail to run airgap update: %v", err)
-	}
-	// remove the airgap bundle and binary after upgrade
-	line = []string{"rm", "/assets/upgrade2/release.airgap"}
-	if _, _, err := tc.RunCommandOnNode(0, line); err != nil {
-		t.Fatalf("fail to remove airgap bundle on node %s: %v", tc.Nodes[0], err)
-	}
-	line = []string{"rm", "/usr/local/bin/embedded-cluster-upgrade2"}
-	if _, _, err := tc.RunCommandOnNode(0, line); err != nil {
-		t.Fatalf("fail to remove embedded-cluster-upgrade2 binary on node %s: %v", tc.Nodes[0], err)
-	}
-
-	appUpgradeVersion = fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA"))
-	testArgs = []string{appUpgradeVersion}
-
-	t.Logf("%s: upgrading cluster a second time", time.Now().Format(time.RFC3339))
-	if stdout, stderr, err := tc.RunPlaywrightTest("deploy-upgrade", testArgs...); err != nil {
-		t.Fatalf("fail to run playwright test deploy-upgrade: %v: %s: %s", err, stdout, stderr)
-	}
-
-	t.Logf("%s: checking installation state after second upgrade", time.Now().Format(time.RFC3339))
-	line = []string{"check-postupgrade-state.sh", k8sVersion(), ecUpgradeTargetVersion()}
-	if stdout, stderr, err := tc.RunCommandOnNode(0, line, withEnv); err != nil {
-		t.Fatalf("fail to check postupgrade state: %v: %s: %s", err, stdout, stderr)
-	}
-
-	// TODO: reset fails with the following error:
-	//   error: could not reset k0s: exit status 1, time="2024-10-17 22:44:52" level=warning msg="To ensure a full reset, a node reboot is recommended."
-	//   Error: errors received during clean-up: [failed to delete /run/k0s. err: unlinkat /run/k0s/containerd/io.containerd.grpc.v1.cri/sandboxes/.../shm: device or resource busy]
-
-	// t.Logf("%s: resetting worker node", time.Now().Format(time.RFC3339))
-	// line = []string{"reset-installation.sh"}
-	// if stdout, stderr, err := tc.RunCommandOnNode(1, line, withEnv); err != nil {
-	// 	t.Fatalf("fail to reset worker node: %v: %s: %s", err, stdout, stderr)
-	// }
-
-	// // use upgrade binary for reset
-	// withUpgradeBin := map[string]string{"EMBEDDED_CLUSTER_BIN": "embedded-cluster-upgrade"}
-
-	// t.Logf("%s: resetting node 0", time.Now().Format(time.RFC3339))
-	// line = []string{"reset-installation.sh"}
-	// if stdout, stderr, err := tc.RunCommandOnNode(0, line, withEnv, withUpgradeBin); err != nil {
-	// 	t.Fatalf("fail to reset node 0: %v: %s: %s", err, stdout, stderr)
-	// }
-
-	t.Logf("%s: test complete", time.Now().Format(time.RFC3339))
-}
-
 func TestMultiNodeAirgapUpgradeSameK0s(t *testing.T) {
 	t.Parallel()
 
@@ -1623,50 +1326,6 @@ func TestMultiNodeAirgapHAInstallation(t *testing.T) {
 	t.Logf("%s: test complete", time.Now().Format(time.RFC3339))
 }
 
-func TestInstallSnapshotFromReplicatedApp(t *testing.T) {
-	t.Parallel()
-
-	RequireEnvVars(t, []string{"SHORT_SHA"})
-
-	tc := docker.NewCluster(&docker.ClusterInput{
-		T:      t,
-		Nodes:  1,
-		Distro: "debian-bookworm",
-	})
-	defer tc.Cleanup()
-
-	downloadECReleaseWithOptions(t, tc, 0, downloadECReleaseOptions{
-		version:   fmt.Sprintf("appver-%s", os.Getenv("SHORT_SHA")),
-		licenseID: SnapshotLicenseID,
-	})
-
-	installSingleNode(t, tc)
-
-	if stdout, stderr, err := tc.SetupPlaywrightAndRunTest("deploy-app"); err != nil {
-		t.Fatalf("fail to run playwright test deploy-app: %v: %s: %s", err, stdout, stderr)
-	}
-
-	checkInstallationState(t, tc)
-
-	t.Logf("%s: ensuring velero is installed", time.Now().Format(time.RFC3339))
-	line := []string{"check-velero-state.sh", os.Getenv("SHORT_SHA")}
-	if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil {
-		t.Fatalf("fail to check velero state: %v: %s: %s", err, stdout, stderr)
-	}
-
-	appUpgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA"))
-	testArgs := []string{appUpgradeVersion}
-
-	t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339))
-	if stdout, stderr, err := tc.RunPlaywrightTest("deploy-upgrade", testArgs...); err != nil {
-		t.Fatalf("fail to run playwright test deploy-upgrade: %v: %s: %s", err, stdout, stderr)
-	}
-
-	checkPostUpgradeState(t, tc)
-
-	t.Logf("%s: test complete", time.Now().Format(time.RFC3339))
-}
-
 func TestSingleNodeInstallationNoopUpgrade(t *testing.T) {
 	t.Parallel()