chore: adding end to end tests (#31)

* chore: adding end to end tests

start adding end to end tests to be run as part of gh actions.

* bug: run post apply directly on localhost

we don't need to use the remote connection when running the post apply
in the current node.

* feat: create no-prompt on token action

two things were implemented here:

- if --no-prompt then dump the installation failure log.
- added --no-prompt on the `token` command.

* bug: setting hostname overwrite on local install

* bug: bring back the ssh implementation

localhost option of k0sctl seems to be buggy, judging by the types of
errors we are seeing in our end to end tests. this commit brings back
the old ssh based implementation, if you are reading this then it means
that it solved the instabilities.

* tests: added token based installation test

added and end to end test to validate the token based cluster creation.

* chore: fixed e2e documentation

* chore: remove dangling sleep call

* chore: using github matrix

* chore: removing unused scripts

* chore: using pipefail

Author: ricardomaraschini

.github/workflows/e2e.yaml

@@ -0,0 +1,43 @@
+name: End to End
+on:
+- pull_request
+jobs:
+  run:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        tests:
+        - TestBuildBundle
+        - TestEmbedAndInstall
+        - TestInstallSingleNodeAndUpgradeToEmbed
+        - TestMultiNodeInstallation
+        - TestSingleNodeInstallation
+        - TestTokenBasedMultiNodeInstallation
+    steps:
+    - name: Move Docker aside
+      run: |
+        sudo systemctl stop docker
+        sudo iptables -F
+        sudo iptables -P FORWARD ACCEPT
+    - name: Checkout
+      uses: actions/checkout@v3
+    - name: Install and Configure LXD
+      run: |
+        sudo lxd init --auto
+        sudo snap set lxd daemon.group=adm
+        sudo snap restart lxd
+        sudo lxd waitready
+    - name: Install and configure OVN
+      run: |
+        sudo apt install ovn-host ovn-central -y
+        sudo ovs-vsctl set open_vswitch .                            \
+             external_ids:ovn-remote=unix:/var/run/ovn/ovnsb_db.sock \
+             external_ids:ovn-encap-type=geneve                      \
+             external_ids:ovn-encap-ip=127.0.0.1
+    - name: Setup Go
+      uses: actions/setup-go@v4
+      with:
+        go-version: "1.21.0"
+    - name: E2E
+      run: |
+        make e2e-test TEST_NAME=${{ matrix.tests }}

.github/workflows/pull-request.yaml

@@ -1,58 +1,55 @@
 name: Pull request
 on:
 - pull_request
-
 jobs:
+  sanitize:
+    name: Sanitize
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+    - name: Setup Go
+      uses: actions/setup-go@v4
+      with:
+        go-version: "1.21.0"
+    - name: Go vet
+      run: |
+        make vet
   tests:
-    name: Test
+    name: Unit tests
     runs-on: ubuntu-latest
     steps:
     - name: Checkout
       uses: actions/checkout@v3
-
     - name: Setup Go
       uses: actions/setup-go@v4
       with:
         go-version: "1.21.0"
-
     - name: Go vet
       run: |
        make vet
-
     - name: Unit tests
       run: |
         make unit-tests
-
-    - name: E2E tests
-      run: |
-        make e2e-tests
-
   build:
     name: Build
     runs-on: ubuntu-latest
-    needs:
-    - tests
     steps:
     - name: Checkout
       uses: actions/checkout@v3
-
     - name: Setup Go
       uses: actions/setup-go@v4
       with:
         go-version: "1.21.0"
-
     - name: Build Linux AMD64
       run: |
         make helmvm-linux-amd64
-
     - name: Build Darwin AMD64
       run: |
         make helmvm-darwin-amd64
-
     - name: Build Darwin ARM64
       run: |
         make helmvm-darwin-arm64
-
     - name: Build HelmVM Builder Server Image
       uses: docker/build-push-action@v4
       with:

.github/workflows/release-dev.yaml

@@ -1,13 +1,10 @@
 name: Release
-
 on:
   push:
     branches:
       - main
-
 permissions:
   contents: write
-
 jobs:
   releaser:
     runs-on: ubuntu-latest
@@ -16,30 +13,25 @@ jobs:
         uses: actions/checkout@v3
         with:
           fetch-depth: 0
-
       - name: Set up Go
         uses: actions/setup-go@v4
         with:
           go-version: "1.20"
-
       - name: Build linux-amd64
         run: |
           make helmvm-linux-amd64
           tar -C output/bin -czvf helmvm-linux-amd64.tgz helmvm
           make clean
-
       - name: Build darwin-amd64
         run: |
           make helmvm-darwin-amd64
           tar -C output/bin -czvf helmvm-darwin-amd64.tgz helmvm
           make clean
-
       - name: Build darwin-arm64
         run: |
           make helmvm-darwin-arm64
           tar -C output/bin -czvf helmvm-darwin-arm64.tgz helmvm
           make clean
-
       - name: Publish development release
         uses: marvinpinto/action-automatic-releases@latest
         with:
@@ -49,7 +41,6 @@ jobs:
           title: Development Release Build
           files: |
             *.tgz
-
       - name: Build HelmVM Builder Server image
         uses: docker/build-push-action@v4
         with:

.github/workflows/release-prod.yaml

@@ -1,13 +1,10 @@
 name: Release
-
 on:
   push:
     tags:
       - "v*.*.*"
-
 permissions:
   contents: write
-
 jobs:
   releaser:
     runs-on: ubuntu-latest
@@ -24,43 +21,36 @@ jobs:
         uses: actions/setup-go@v4
         with:
           go-version: "1.20"
-
       - name: Build linux-amd64
         run: |
           make helmvm-linux-amd64
           tar -C output/bin -czvf helmvm-linux-amd64.tgz helmvm
           make clean
-
       - name: Build darwin-amd64
         run: |
           make helmvm-darwin-amd64
           tar -C output/bin -czvf helmvm-darwin-amd64.tgz helmvm
           make clean
-
       - name: Build darwin-arm64
         run: |
           make helmvm-darwin-arm64
           tar -C output/bin -czvf helmvm-darwin-arm64.tgz helmvm
           make clean
-
       - name: Publish release
         uses: marvinpinto/action-automatic-releases@latest
         with:
           repo_token: ${{ secrets.GITHUB_TOKEN }}
           prerelease: false
           files: |
             *.tgz
-
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
-
       - name: Login to GitHub Container Registry
         uses: docker/login-action@v2
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
-
       - name: Build and Publish HelmVM Builder Server image.
         uses: docker/build-push-action@v4
         with:

Makefile

@@ -125,15 +125,29 @@ builder: static static-linux-amd64
 
 .PHONY: unit-tests
 unit-tests:
-	go test -v ./...
+	go test -v $(shell go list ./... | grep -v /e2e)
 
 .PHONY: vet
 vet: static-linux-amd64 static
 	go vet ./...
 
 .PHONY: e2e-tests
-e2e-tests:
-	echo to be implemented
+e2e-tests: helmvm-linux-amd64
+	mkdir -p output/tmp
+	rm -rf output/tmp/id_rsa*
+	ssh-keygen -t rsa -N "" -C "Integration Test Key" -f output/tmp/id_rsa
+	go test -timeout 45m -parallel 1 -failfast -v ./e2e
+
+.PHONY: e2e-test
+e2e-test: helmvm-linux-amd64
+	mkdir -p output/tmp
+	rm -rf output/tmp/id_rsa*
+	ssh-keygen -t rsa -N "" -C "Integration Test Key" -f output/tmp/id_rsa
+	go test -timeout 10m -v ./e2e -run $(TEST_NAME)
+
+.PHONY: create-e2e-workflows
+create-e2e-workflows:
+	./e2e/hack/create-e2e-workflows.sh
 
 .PHONY: clean
 clean:

cmd/helmvm/install.go

@@ -45,7 +45,7 @@ func runPostApply(ctx context.Context) error {
 		return fmt.Errorf("unable to read cluster config: %w", err)
 	}
 	for _, host := range cfg.Spec.Hosts {
-		if err := runPostApplyOnHost(ctx, host, logger); err != nil {
+		if err := runPostApplyOnHost(ctx, host); err != nil {
 			return err
 		}
 	}
@@ -55,7 +55,7 @@ func runPostApply(ctx context.Context) error {
 // runPostApply runs the post-apply script on a host. XXX I don't think this
 // belongs here and needs to be refactored in a more generic way. It's here
 // because I have other things to do and this is a prototype.
-func runPostApplyOnHost(ctx context.Context, host *cluster.Host, logger pb.MessageWriter) error {
+func runPostApplyOnHost(ctx context.Context, host *cluster.Host) error {
 	if err := host.Connect(); err != nil {
 		return fmt.Errorf("failed to connect to host: %w", err)
 	}
@@ -280,6 +280,10 @@ func applyK0sctl(c *cli.Context, prompt bool, nodes []infra.Node) error {
 	logrus.Infof("Applying cluster configuration")
 	if err := runK0sctlApply(c.Context); err != nil {
 		logrus.Errorf("Installation or upgrade failed.")
+		if !prompt {
+			dumpApplyLogs()
+			return fmt.Errorf("unable to apply cluster: %w", err)
+		}
 		var useCurrent = &survey.Confirm{
 			Message: "Do you wish to visualize the logs?",
 			Default: true,

cmd/helmvm/token.go

@@ -36,6 +36,11 @@ var tokenCreateCommand = &cli.Command{
 			Usage: "For how long the token should be valid",
 			Value: 24 * time.Hour,
 		},
+		&cli.BoolFlag{
+			Name:  "no-prompt",
+			Usage: "Do not prompt user when it is not necessary",
+			Value: false,
+		},
 	},
 	Action: func(c *cli.Context) error {
 		if runtime.GOOS != "linux" {
@@ -48,6 +53,7 @@ var tokenCreateCommand = &cli.Command{
 		if role != "worker" && role != "controller" {
 			return fmt.Errorf("invalid role %q", role)
 		}
+		prompt := !c.Bool("no-prompt")
 		cfgpath := defaults.PathToConfig("k0sctl.yaml")
 		if _, err := os.Stat(cfgpath); err != nil {
 			if os.IsNotExist(err) {
@@ -63,14 +69,17 @@ var tokenCreateCommand = &cli.Command{
 			logrus.Warn("Through the centralized management you can manage all your")
 			logrus.Warn("cluster nodes from a single location. If you decide to move")
 			logrus.Warn("on the centralized management won't be available anymore")
-			question := &survey.Confirm{
-				Message: "Do you want to use continue ?", Default: true,
-			}
-			var moveOn bool
-			if err := survey.AskOne(question, &moveOn); err != nil {
-				return fmt.Errorf("unable to ask for user confirmation: %w", err)
-			} else if !moveOn {
-				return nil
+			if prompt {
+				question := &survey.Confirm{
+					Message: "Do you want to use continue ?",
+					Default: true,
+				}
+				var moveOn bool
+				if err := survey.AskOne(question, &moveOn); err != nil {
+					return fmt.Errorf("unable to ask for confirmation: %w", err)
+				} else if !moveOn {
+					return nil
+				}
 			}
 		}
 		dur := c.Duration("expiry").String()
@@ -90,7 +99,7 @@ var tokenCreateCommand = &cli.Command{
 		logrus.Infof("This token is valid for %s hours.", dur)
 		logrus.Infof("You can now run the following command in a remote node to add it")
 		logrus.Infof("to the cluster as a %q node:", role)
-		fmt.Printf("%s node join --role %q %s", defaults.BinaryName(), role, buf.String())
+		fmt.Printf("%s node join --role %s %s", defaults.BinaryName(), role, buf.String())
 		return nil
 	},
 }