feat: e2e tests for multi node clusters (#285)

* feat: add script to getting join token

adds a script to be used in our e2e tests. this script generates a join
token for a controller join token.

```bash
scripts: ./generate-controller-join-token.js 192.168.86.6
opening a new tab
acessing kotsadm on port 30000
waiting and clickin on the 'Continue to Setup' button
waiting and clicking on 'Advanced' to move on with the certificate
waiting and clicking on 'Proceed' to move on with the certificate
going to the /tls endpoint
waiting and clicking on 'Continue'
waiting and clicking in the password field
typing the password
clicking in the Log in button
waiting and clicking in the Cluster Management tab
waiting and clicking in the Add node button
waiting and clicking in the controller role
waiting and fetching the node join command
sudo ./ec node join 192.168.86.6:30000 59hKbnRkZmAbFFbkZ8WoOGZX
scripts:
```

* chore: adding script for worker and controller join tokens

* chore: add e2e test for multi node clusters

Author: ricardomaraschini

.github/workflows/e2e.yaml

@@ -17,6 +17,7 @@ jobs:
         - TestInstallWithDisabledAddons
         - TestHostPreflight
         - TestUnsupportedOverrides
+        - TestMultiNodeInstallation
     steps:
     - name: Move Docker aside
       run: |

e2e/install_test.go

@@ -2,11 +2,22 @@ package e2e
 
 import (
 	"encoding/json"
+	"strings"
 	"testing"
+	"time"
 
 	"github.com/replicatedhq/embedded-cluster/e2e/cluster"
 )
 
+type clusterStatusResponse struct {
+	App     string `json:"app"`
+	Cluster string `json:"cluster"`
+}
+
+type nodeJoinResponse struct {
+	Command string `json:"command"`
+}
+
 func TestSingleNodeInstallation(t *testing.T) {
 	t.Parallel()
 	tc := cluster.NewTestCluster(&cluster.Input{
@@ -48,11 +59,7 @@ func TestSingleNodeInstallation(t *testing.T) {
 		t.Log("stderr:", stderr)
 		t.Fatalf("fail to access kotsadm interface and state: %v", err)
 	}
-	type response struct {
-		App     string `json:"app"`
-		Cluster string `json:"cluster"`
-	}
-	var r response
+	var r clusterStatusResponse
 	if err := json.Unmarshal([]byte(stdout), &r); err != nil {
 		t.Log("stdout:", stdout)
 		t.Log("stderr:", stderr)
@@ -246,3 +253,121 @@ func TestHostPreflight(t *testing.T) {
 		t.Fatalf("fail to install embedded-cluster on node %s: %v", tc.Nodes[0], err)
 	}
 }
+
+// This test creates 4 nodes, installs on the first one and then generate 2 join tokens
+// for controllers and one join token for worker nodes. Joins the nodes and then waits
+// for them to report ready.
+func TestMultiNodeInstallation(t *testing.T) {
+	tc := cluster.NewTestCluster(&cluster.Input{
+		T:                   t,
+		Nodes:               4,
+		Image:               "ubuntu/jammy",
+		SSHPublicKey:        "../output/tmp/id_rsa.pub",
+		SSHPrivateKey:       "../output/tmp/id_rsa",
+		EmbeddedClusterPath: "../output/bin/embedded-cluster",
+	})
+	defer tc.Destroy()
+	t.Log("installing ssh on node 0")
+	commands := [][]string{{"apt-get", "update", "-y"}, {"apt-get", "install", "openssh-server", "-y"}}
+	if err := RunCommandsOnNode(t, tc, 0, commands); err != nil {
+		t.Fatalf("fail to install ssh on node %s: %v", tc.Nodes[0], err)
+	}
+
+	// bootstrap the first node and makes sure it is healthy. also executes the kots
+	// ssl certificate configuration (kurl-proxy).
+	t.Log("installing embedded-cluster on node 0")
+	if stdout, stderr, err := RunCommandOnNode(t, tc, 0, []string{"single-node-install.sh"}); err != nil {
+		t.Logf("stdout: %s\nstderr: %s", stdout, stderr)
+		t.Fatalf("fail to install embedded-cluster on node %s: %v", tc.Nodes[0], err)
+	}
+	t.Log("installing puppeteer on node 0")
+	if stdout, stderr, err := RunCommandOnNode(t, tc, 0, []string{"install-puppeteer.sh"}); err != nil {
+		t.Logf("stdout: %s\nstderr: %s", stdout, stderr)
+		t.Fatalf("fail to install puppeteer on node %s: %v", tc.Nodes[0], err)
+	}
+	t.Log("accessing kotsadm interface and checking app and cluster state")
+	line := []string{"puppeteer.sh", "check-app-and-cluster-status.js", "10.0.0.2"}
+	stdout, stderr, err := RunCommandOnNode(t, tc, 0, line)
+	if err != nil {
+		t.Logf("stdout: %s\nstderr: %s", stdout, stderr)
+		t.Fatalf("fail to access kotsadm interface and state: %v", err)
+	}
+	var r clusterStatusResponse
+	if err := json.Unmarshal([]byte(stdout), &r); err != nil {
+		t.Logf("stdout: %s\nstderr: %s", stdout, stderr)
+		t.Fatalf("fail to parse script response: %v", err)
+	} else if r.App != "Ready" || r.Cluster != "Up to date" {
+		t.Logf("stdout: %s\nstderr: %s", stdout, stderr)
+		t.Fatalf("cluster or app not ready: %s", stdout)
+	}
+
+	// generate all node join commands (2 for controllers and 1 for worker).
+	t.Log("generating two new controller token commands")
+	controllerCommands := []string{}
+	for i := 0; i < 2; i++ {
+		line = []string{"puppeteer.sh", "generate-controller-join-token.js", "10.0.0.2"}
+		stdout, stderr, err := RunCommandOnNode(t, tc, 0, line)
+		if err != nil {
+			t.Logf("stdout: %s\nstderr: %s", stdout, stderr)
+			t.Fatalf("fail to generate controller join token: %s", stdout)
+		}
+		var r nodeJoinResponse
+		if err := json.Unmarshal([]byte(stdout), &r); err != nil {
+			t.Logf("stdout: %s\nstderr: %s", stdout, stderr)
+			t.Fatalf("fail to parse script response: %v", err)
+		}
+		// trim down the "./" and the "sudo" command as those are not needed. we run as
+		// root and the embedded-cluster binary is on the PATH.
+		command := strings.TrimPrefix(r.Command, "sudo ./")
+		controllerCommands = append(controllerCommands, command)
+		t.Log("controller join token command:", command)
+	}
+	t.Log("generating a new worker token command")
+	line = []string{"puppeteer.sh", "generate-worker-join-token.js", "10.0.0.2"}
+	stdout, stderr, err = RunCommandOnNode(t, tc, 0, line)
+	if err != nil {
+		t.Logf("stdout: %s\nstderr: %s", stdout, stderr)
+		t.Fatalf("fail to generate controller join token: %s", stdout)
+	}
+	var jr nodeJoinResponse
+	if err := json.Unmarshal([]byte(stdout), &jr); err != nil {
+		t.Logf("stdout: %s\nstderr: %s", stdout, stderr)
+		t.Fatalf("fail to parse script response: %v", err)
+	}
+
+	// join the nodes.
+	for i, cmd := range controllerCommands {
+		node := i + 1
+		t.Logf("joining node %d to the cluster (controller)", node)
+		stdout, stderr, err := RunCommandOnNode(t, tc, node, strings.Split(cmd, " "))
+		if err != nil {
+			t.Logf("stdout: %s\nstderr: %s", stdout, stderr)
+			t.Fatalf("fail to join node %d as a controller: %v", node, err)
+		}
+		// XXX If we are too aggressive joining nodes we can see the following error being
+		// thrown by kotsadm on its log (and we get a 500 back):
+		// "
+		// failed to get controller role name: failed to get cluster config: failed to get
+		// current installation: failed to list installations: etcdserver: leader changed
+		// "
+		t.Logf("node %d joined, sleeping...", node)
+		time.Sleep(30 * time.Second)
+	}
+	command := strings.TrimPrefix(jr.Command, "sudo ./")
+	t.Log("worker join token command:", command)
+	t.Log("joining node 3 to the cluster as a worker")
+	stdout, stderr, err = RunCommandOnNode(t, tc, 3, strings.Split(command, " "))
+	if err != nil {
+		t.Logf("stdout: %s\nstderr: %s", stdout, stderr)
+		t.Fatalf("fail to join node 3 to the cluster as a worker: %v", err)
+	}
+
+	// wait for the nodes to report as ready.
+	t.Log("all nodes joined, waiting for them to be ready")
+	stdout, stderr, err = RunCommandOnNode(t, tc, 0, []string{"wait-for-ready-nodes.sh", "4"})
+	if err != nil {
+		t.Logf("stdout: %s\nstderr: %s", stdout, stderr)
+		t.Fatalf("fail to install embedded-cluster on node %s: %v", tc.Nodes[0], err)
+	}
+	t.Log(stdout)
+}

e2e/scripts/check-app-and-cluster-status.js

@@ -0,0 +1,180 @@
+#!/usr/bin/env node
+
+/*
+ * this script has been generated with chrome recorder and then pasted here.
+ * some parts were manually changed, these are flagged with a CUSTOM comment.
+ * all logging has also been manually added (process.stderr.write() calls).
+ * this script is meant to be run as an argument to the `puppeteer.sh` script.
+ * THIS SCRIPT EXPECTS THE STEP TO ENABLE HTTPS ACCESS TO KOTS TO BE ALREADY
+ * COMPLETED. YOU NEED TO RUN check-app-and-cluster-status.js BEFORE THIS.
+ */
+
+const puppeteer = require('puppeteer'); // v20.7.4 or later
+
+(async () => {
+    const browser = await puppeteer.launch(
+        {
+            headless: 'new',
+            // CUSTOM: added the following line to fix the "No usable sandbox!" error.
+            args: ['--no-sandbox', '--disable-setuid-sandbox'],
+            // CUSTOM: added ignore https errors.
+            ignoreHTTPSErrors: true
+        }
+    );
+    const page = await browser.newPage();
+    const timeout = 5000;
+    page.setDefaultTimeout(timeout);
+    const args = process.argv.slice(2);
+    if (args.length !== 1) {
+        throw new Error('usage: generate-controller-join-token.js <kotsadm-ip>');
+    }
+
+    {
+        const targetPage = page;
+        await targetPage.setViewport({
+            width: 1512,
+            height: 761
+        })
+    }
+    {
+        process.stderr.write("opening a new tab\n");
+        const targetPage = page;
+        const promises = [];
+        const startWaitingForEvents = () => {
+            promises.push(targetPage.waitForNavigation());
+        }
+        startWaitingForEvents();
+        await targetPage.goto('chrome://new-tab-page/');
+        await Promise.all(promises);
+    }
+    {
+        process.stderr.write("acessing kotsadm on port 30000 (HTTPS)\n");
+        const targetPage = page;
+        const promises = [];
+        const startWaitingForEvents = () => {
+            promises.push(targetPage.waitForNavigation());
+        }
+        startWaitingForEvents();
+        // CUSTOM: using the command line argument.
+        await targetPage.goto(`https://${args[0]}:30000/`);
+        await Promise.all(promises);
+    }
+    {
+        process.stderr.write("waiting and clicking in the password field\n");
+        const targetPage = page;
+        await puppeteer.Locator.race([
+            targetPage.locator('::-p-aria(password)'),
+            targetPage.locator('input'),
+            targetPage.locator('::-p-xpath(//*[@id=\\"app\\"]/div/div[2]/div/div/div/div[2]/div/div/div[1]/input)'),
+            targetPage.locator(':scope >>> input')
+        ])
+            .setTimeout(timeout)
+            .click({
+              offset: {
+                x: 80,
+                y: 21.0078125,
+              },
+            });
+    }
+    {
+        process.stderr.write("typing the password\n");
+        const targetPage = page;
+        await puppeteer.Locator.race([
+            targetPage.locator('::-p-aria(password)'),
+            targetPage.locator('input'),
+            targetPage.locator('::-p-xpath(//*[@id=\\"app\\"]/div/div[2]/div/div/div/div[2]/div/div/div[1]/input)'),
+            targetPage.locator(':scope >>> input')
+        ])
+            .setTimeout(timeout)
+            .fill('password');
+    }
+    {
+        process.stderr.write("clicking in the Log in button\n");
+        const targetPage = page;
+        await puppeteer.Locator.race([
+            targetPage.locator('::-p-aria(Log in)'),
+            targetPage.locator('button'),
+            targetPage.locator('::-p-xpath(//*[@id=\\"app\\"]/div/div[2]/div/div/div/div[2]/div/div/div[2]/button)'),
+            targetPage.locator(':scope >>> button')
+        ])
+            .setTimeout(timeout)
+            .click({
+              offset: {
+                x: 30,
+                y: 14.0078125,
+              },
+            });
+    }
+    {
+        process.stderr.write("waiting and clicking in the Cluster Management tab\n");
+        const targetPage = page;
+        await puppeteer.Locator.race([
+            targetPage.locator('div:nth-of-type(3) > span'),
+            targetPage.locator('::-p-xpath(//*[@id=\\"app\\"]/div/div[1]/div[1]/div[2]/div[3]/span)'),
+            targetPage.locator(':scope >>> div:nth-of-type(3) > span'),
+            targetPage.locator('::-p-text(Cluster Management)')
+        ])
+            .setTimeout(timeout)
+            .click({
+              offset: {
+                x: 108.734375,
+                y: 28,
+              },
+            });
+    }
+    {
+        process.stderr.write("waiting and clicking in the Add node button\n");
+        const targetPage = page;
+        await puppeteer.Locator.race([
+            targetPage.locator('::-p-aria(Add node)'),
+            targetPage.locator('div.tw-flex > button'),
+            targetPage.locator('::-p-xpath(//*[@id=\\"app\\"]/div/div[2]/div/div/div[1]/button)'),
+            targetPage.locator(':scope >>> div.tw-flex > button'),
+            targetPage.locator('::-p-text(Add node)')
+        ])
+            .setTimeout(timeout)
+            .click({
+              offset: {
+                x: 16.328125,
+                y: 13,
+              },
+            });
+    }
+    {
+        process.stderr.write("waiting and clicking in the controller role\n");
+        const targetPage = page;
+        await puppeteer.Locator.race([
+            targetPage.locator('div:nth-of-type(1) > label'),
+            targetPage.locator('::-p-xpath(/html/body/div[5]/div/div/div/div[2]/div[1]/label)'),
+            targetPage.locator(':scope >>> div:nth-of-type(1) > label')
+        ])
+            .setTimeout(timeout)
+            .click({
+              offset: {
+                x: 110,
+                y: 27.5,
+              },
+            });
+    }
+    {
+        // CUSTOM: finding the element that contains the node join command.
+        process.stderr.write("waiting and fetching the node join command\n");
+        const targetPage = page;
+        await targetPage.waitForSelector('.react-prism.language-bash');
+        let elementContent = await targetPage.evaluate(() => {
+            const element = document.querySelector('.react-prism.language-bash');
+            return element ? element.textContent : null;
+        });
+        if (!elementContent) {
+            throw new Error("Could not find the node join command");
+        }
+        let result = { command: elementContent };
+        console.log(JSON.stringify(result));
+    }
+
+    await browser.close();
+
+})().catch(err => {
+    console.error(err);
+    process.exit(1);
+});