chore: adding unit tests on infra package (#58)

ricardomaraschini · web-flow · commit 3fae97e634b3 · 2023-09-08T15:05:48.000+02:00
* chore: adding unit tests on infra package

adding some basic unit tests on infra package.

* chore: fix some flaky e2e tests

some e2e tests were failing due to lack of disk space in the github
runner. this commit hopefully solves all of them.
diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
@@ -5,6 +5,7 @@ jobs:
   run:
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
         tests:
         - TestBuildBundle
@@ -45,6 +46,11 @@ jobs:
       uses: actions/setup-go@v4
       with:
         go-version: "1.21.0"
+    - name: Free up runner disk space
+      run: |
+        sudo rm -rf /usr/local/lib/android
+        sudo rm -rf /opt/hostedtoolcache/CodeQL
+        sudo rm -rf /opt/hostedtoolcache/Python
     - name: E2E
       run: |
         make e2e-test TEST_NAME=${{ matrix.tests }}
diff --git a/Makefile b/Makefile
@@ -154,7 +154,7 @@ e2e-test: helmvm-linux-amd64
 	mkdir -p output/tmp
 	rm -rf output/tmp/id_rsa*
 	ssh-keygen -t rsa -N "" -C "Integration Test Key" -f output/tmp/id_rsa
-	go test -timeout 30m -v ./e2e -run $(TEST_NAME)$
+	go test -timeout 45m -v ./e2e -run $(TEST_NAME)$
 
 .PHONY: create-e2e-workflows
 create-e2e-workflows:
diff --git a/e2e/embed_test.go b/e2e/embed_test.go
@@ -27,11 +27,6 @@ func TestEmbedAndInstall(t *testing.T) {
 	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
 		t.Fatalf("fail to install embedded ssh in node 0: %v", err)
 	}
-	t.Log("creating deployment mounting pvc")
-	line = []string{"deploy-with-pvc.sh"}
-	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
-		t.Fatalf("fail to create deployment with pvc: %v", err)
-	}
 }
 
 func TestEmbedAddonsOnly(t *testing.T) {
diff --git a/e2e/install_test.go b/e2e/install_test.go
@@ -87,11 +87,6 @@ func TestSingleNodeInstallation(t *testing.T) {
 	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
 		t.Fatalf("fail to install helmvm on node %s: %v", tc.Nodes[0], err)
 	}
-	t.Log("creating deployment mounting pvc")
-	line = []string{"deploy-with-pvc.sh"}
-	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
-		t.Fatalf("fail to create deployment with pvc: %v", err)
-	}
 }
 
 func TestMultiNodeInstallation(t *testing.T) {
@@ -145,11 +140,6 @@ func TestSingleNodeInstallationRockyLinux8(t *testing.T) {
 	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
 		t.Fatalf("fail to install helmvm on node %s: %v", tc.Nodes[0], err)
 	}
-	t.Log("creating deployment mounting pvc")
-	line = []string{"deploy-with-pvc.sh"}
-	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
-		t.Fatalf("fail to create deployment with pvc: %v", err)
-	}
 }
 
 func TestSingleNodeInstallationDebian12(t *testing.T) {
@@ -174,10 +164,6 @@ func TestSingleNodeInstallationDebian12(t *testing.T) {
 		t.Fatalf("fail to install helmvm on node %s: %v", tc.Nodes[0], err)
 	}
 	t.Log("creating deployment mounting pvc")
-	line = []string{"deploy-with-pvc.sh"}
-	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
-		t.Fatalf("fail to create deployment with pvc: %v", err)
-	}
 }
 
 func TestSingleNodeInstallationCentos8Stream(t *testing.T) {
@@ -205,11 +191,6 @@ func TestSingleNodeInstallationCentos8Stream(t *testing.T) {
 	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
 		t.Fatalf("fail to install helmvm on node %s: %v", tc.Nodes[0], err)
 	}
-	t.Log("creating deployment mounting pvc")
-	line = []string{"deploy-with-pvc.sh"}
-	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
-		t.Fatalf("fail to create deployment with pvc: %v", err)
-	}
 }
 
 func TestMultiNodeInteractiveInstallation(t *testing.T) {
diff --git a/e2e/scripts/addons-only.sh b/e2e/scripts/addons-only.sh
@@ -53,16 +53,16 @@ embed_helm_chart() {
 }
 
 wait_for_memcached_pods() {
-    ready=$(kubectl get pods -n helmvm | grep -v NotReady | grep Ready | grep -c memcached || true)
+    ready=$(kubectl get pods -n helmvm | grep -c memcached || true)
     counter=0
     while [ -z "$ready" ] || [ "$ready" -lt "1" ]; do
         if [ "$counter" -gt 36 ]; then
             return 1
         fi
         sleep 5
         counter=$((counter+1))
-        echo "Waiting for memcached pods to be ready"
-        ready=$(kubectl get pods -n helmvm | grep  Running | grep -c memcached || true)
+        echo "Waiting for memcached pods"
+        ready=$(kubectl get pods -n helmvm | grep -c memcached || true)
         kubectl get pods -n helmvm 2>&1 || true
         echo "$ready"
     done
@@ -97,7 +97,7 @@ main() {
     fi
     echo "waiting for memcached " >> /tmp/log
     if ! wait_for_memcached_pods; then
-        echo "Memcached pods not reporting healthy"
+        echo "Memcached pods not appearing"
         exit 1
     fi
 }
diff --git a/e2e/scripts/deploy-with-pvc.sh b/e2e/scripts/deploy-with-pvc.sh
diff --git a/e2e/scripts/embed-and-install.sh b/e2e/scripts/embed-and-install.sh
@@ -54,22 +54,50 @@ embed_helm_chart() {
 }
 
 wait_for_memcached_pods() {
-    ready=$(kubectl get pods -n helmvm | grep -v NotReady | grep Ready | grep -c memcached || true)
+    ready=$(kubectl get pods -n helmvm | grep -c memcached || true)
     counter=0
     while [ "$ready" -lt "1" ]; do
         if [ "$counter" -gt 36 ]; then
             return 1
         fi
         sleep 5
         counter=$((counter+1))
-        echo "Waiting for memcached pods to be ready"
-        ready=$(kubectl get pods -n helmvm | grep  Running | grep -c memcached || true)
+        echo "Waiting for memcached pods"
+        ready=$(kubectl get pods -n helmvm | grep -c memcached || true)
         kubectl get pods -n helmvm 2>&1 || true
         echo "$ready"
     done
     return 0
 }
 
+wait_for_pods_running() {
+    local timeout="$1"
+    local start_time
+    local current_time
+    local elapsed_time
+    start_time=$(date +%s)
+    while true; do
+        current_time=$(date +%s)
+        elapsed_time=$((current_time - start_time))
+        if [ "$elapsed_time" -ge "$timeout" ]; then
+            kubectl get pods -A -o yaml || true
+            kubectl describe nodes || true
+            echo "Timed out waiting for all pods to be running."
+            return 1
+        fi
+        local non_running_pods
+        non_running_pods=$(kubectl get pods --all-namespaces --no-headers 2>/dev/null | awk '$4 != "Running" && $4 != "Completed" { print $0 }' | wc -l || echo 1)
+        if [ "$non_running_pods" -ne 0 ]; then
+            echo "Not all pods are running. Waiting."
+            kubectl get pods,nodes -A || true
+            sleep 5
+            continue
+        fi
+        echo "All pods are running."
+        return 0
+    done
+}
+
 main() {
     if ! install_helm ; then
         echo "Failed to install helm"
@@ -96,9 +124,13 @@ main() {
         echo "Nodes not reporting healthy"
         exit 1
     fi
+    if ! wait_for_pods_running 900; then
+        echo "Pods not running"
+        exit 1
+    fi
     echo "waiting for memcached " >> /tmp/log
     if ! wait_for_memcached_pods; then
-        echo "Memcached pods not reporting healthy"
+        echo "Memcached pods not present"
         exit 1
     fi
 }
diff --git a/e2e/scripts/single-node-install.sh b/e2e/scripts/single-node-install.sh
@@ -17,6 +17,34 @@ wait_for_healthy_node() {
     return 0
 }
 
+wait_for_pods_running() {
+    local timeout="$1"
+    local start_time
+    local current_time
+    local elapsed_time
+    start_time=$(date +%s)
+    while true; do
+        current_time=$(date +%s)
+        elapsed_time=$((current_time - start_time))
+        if [ "$elapsed_time" -ge "$timeout" ]; then
+            kubectl get pods -A -o yaml || true
+            kubectl describe nodes || true
+            echo "Timed out waiting for all pods to be running."
+            return 1
+        fi
+        local non_running_pods
+        non_running_pods=$(kubectl get pods --all-namespaces --no-headers 2>/dev/null | awk '$4 != "Running" && $4 != "Completed" { print $0 }' | wc -l || echo 1)
+        if [ "$non_running_pods" -ne 0 ]; then
+            echo "Not all pods are running. Waiting."
+            kubectl get pods,nodes -A || true
+            sleep 5
+            continue
+        fi
+        echo "All pods are running."
+        return 0
+    done
+}
+
 main() {
     if ! helmvm install --no-prompt 2>&1 | tee /tmp/log ; then
         cat /etc/os-release
@@ -31,6 +59,10 @@ main() {
         echo "Failed to install helmvm"
         exit 1
     fi
+    if ! wait_for_pods_running 900; then
+        echo "Failed to install helmvm"
+        exit 1
+    fi
 }
 
 export KUBECONFIG=/root/.helmvm/etc/kubeconfig
diff --git a/e2e/utils.go b/e2e/utils.go
@@ -38,7 +38,7 @@ func RunCommandOnNode(t *testing.T, cl *cluster.Output, node int, line []string)
 		Stdout: stdout,
 		Stderr: stderr,
 	}
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
 	defer cancel()
 	if err := cluster.Run(ctx, t, cmd); err != nil {
 		t.Logf("stdout:\n%s", stdout.String())
diff --git a/pkg/infra/infra.go b/pkg/infra/infra.go
diff --git a/pkg/infra/infra_test.go b/pkg/infra/infra_test.go

Original file line number	Diff line number	Diff line change
`@@ -27,11 +27,6 @@ func TestEmbedAndInstall(t *testing.T) {`
`27`	`27`	`if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {`
`28`	`28`	`t.Fatalf("fail to install embedded ssh in node 0: %v", err)`
`29`	`29`	`}`
`30`		`- t.Log("creating deployment mounting pvc")`
`31`		`- line = []string{"deploy-with-pvc.sh"}`
`32`		`- if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {`
`33`		`- t.Fatalf("fail to create deployment with pvc: %v", err)`
`34`		`- }`
`35`	`30`	`}`
`36`	`31`
`37`	`32`	`func TestEmbedAddonsOnly(t *testing.T) {`
Original file line number	Diff line number	Diff line change
`@@ -87,11 +87,6 @@ func TestSingleNodeInstallation(t *testing.T) {`
`87`	`87`	`if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {`
`88`	`88`	`t.Fatalf("fail to install helmvm on node %s: %v", tc.Nodes[0], err)`
`89`	`89`	`}`
`90`		`- t.Log("creating deployment mounting pvc")`
`91`		`- line = []string{"deploy-with-pvc.sh"}`
`92`		`- if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {`
`93`		`- t.Fatalf("fail to create deployment with pvc: %v", err)`
`94`		`- }`
`95`	`90`	`}`
`96`	`91`
`97`	`92`	`func TestMultiNodeInstallation(t *testing.T) {`
`@@ -145,11 +140,6 @@ func TestSingleNodeInstallationRockyLinux8(t *testing.T) {`
`145`	`140`	`if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {`
`146`	`141`	`t.Fatalf("fail to install helmvm on node %s: %v", tc.Nodes[0], err)`
`147`	`142`	`}`
`148`		`- t.Log("creating deployment mounting pvc")`
`149`		`- line = []string{"deploy-with-pvc.sh"}`
`150`		`- if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {`
`151`		`- t.Fatalf("fail to create deployment with pvc: %v", err)`
`152`		`- }`
`153`	`143`	`}`
`154`	`144`
`155`	`145`	`func TestSingleNodeInstallationDebian12(t *testing.T) {`
`@@ -174,10 +164,6 @@ func TestSingleNodeInstallationDebian12(t *testing.T) {`
`174`	`164`	`t.Fatalf("fail to install helmvm on node %s: %v", tc.Nodes[0], err)`
`175`	`165`	`}`
`176`	`166`	`t.Log("creating deployment mounting pvc")`
`177`		`- line = []string{"deploy-with-pvc.sh"}`
`178`		`- if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {`
`179`		`- t.Fatalf("fail to create deployment with pvc: %v", err)`
`180`		`- }`
`181`	`167`	`}`
`182`	`168`
`183`	`169`	`func TestSingleNodeInstallationCentos8Stream(t *testing.T) {`
`@@ -205,11 +191,6 @@ func TestSingleNodeInstallationCentos8Stream(t *testing.T) {`
`205`	`191`	`if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {`
`206`	`192`	`t.Fatalf("fail to install helmvm on node %s: %v", tc.Nodes[0], err)`
`207`	`193`	`}`
`208`		`- t.Log("creating deployment mounting pvc")`
`209`		`- line = []string{"deploy-with-pvc.sh"}`
`210`		`- if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {`
`211`		`- t.Fatalf("fail to create deployment with pvc: %v", err)`
`212`		`- }`
`213`	`194`	`}`
`214`	`195`
`215`	`196`	`func TestMultiNodeInteractiveInstallation(t *testing.T) {`
Original file line number	Diff line number	Diff line change
`@@ -38,7 +38,7 @@ func RunCommandOnNode(t testing.T, cl cluster.Output, node int, line []string)`
`38`	`38`	`Stdout: stdout,`
`39`	`39`	`Stderr: stderr,`
`40`	`40`	`}`
`41`		`- ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)`
	`41`	`+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)`
`42`	`42`	`defer cancel()`
`43`	`43`	`if err := cluster.Run(ctx, t, cmd); err != nil {`
`44`	`44`	`t.Logf("stdout:\n%s", stdout.String())`