ci: improve test stability with extended timeouts and better error handling

- Increase test timeout from 30m to 60m in GNUmakefile
- Add 90-minute job timeouts to GitHub Actions workflow
- Improve CloudStack simulator readiness check with better logging
- Add retry logic for data center deployment (3 attempts)
- Add detailed error reporting for failed simulator startup

This addresses the frequent CI timeouts around 27-29 minutes by:
1. Giving tests more time to complete (60m vs 30m)
2. Allowing jobs to run longer (90m vs default 6h)
3. Better debugging info when simulator fails to start
4. Retry mechanism for transient deployment failures

Author: sidshas03

.github/actions/setup-cloudstack/action.yml

@@ -43,18 +43,30 @@ runs:
       run: |
         echo "Starting Cloudstack health check"
         T=0
-        until [ $T -gt 40 ]  || curl -sfL http://localhost:8080 --output /dev/null
+        MAX_WAIT=40
+        SLEEP_INTERVAL=30
+        
+        echo "Will wait up to $((MAX_WAIT * SLEEP_INTERVAL / 60)) minutes for CloudStack to be ready"
+        
+        until [ $T -gt $MAX_WAIT ] || curl -sfL http://localhost:8080 --output /dev/null
         do
-            echo "Waiting for Cloudstack to be ready..."
+            echo "Waiting for Cloudstack to be ready... (attempt $((T+1))/$((MAX_WAIT+1)), elapsed: $((T * SLEEP_INTERVAL / 60))m $((T * SLEEP_INTERVAL % 60))s)"
             ((T+=1))
-            sleep 30
+            sleep $SLEEP_INTERVAL
         done
 
         # After loop, check if Cloudstack is up
         if ! curl -sfSL http://localhost:8080 --output /dev/null; then
-          echo "Cloudstack did not become ready in time"
+          echo "Cloudstack did not become ready in time after $((MAX_WAIT * SLEEP_INTERVAL / 60)) minutes"
+          echo "Checking CloudStack container status:"
+          docker ps -a | grep cloudstack || echo "No CloudStack containers found"
+          echo "Checking CloudStack logs:"
+          docker logs $(docker ps -q --filter "ancestor=apache/cloudstack-simulator") --tail 50 || echo "Could not retrieve logs"
+          echo "Testing connectivity:"
           curl -v http://localhost:8080 || true
           exit 22
+        else
+          echo "CloudStack is ready after $((T * SLEEP_INTERVAL / 60))m $((T * SLEEP_INTERVAL % 60))s"
         fi
     - name: Setting up Cloudstack
       id: setup-cloudstack
@@ -64,8 +76,23 @@ runs:
         set -euo pipefail
 
         echo "Deploying Data Center..."
-        docker exec $(docker container ls --format=json -l | jq -r .ID) \
-          python /root/tools/marvin/marvin/deployDataCenter.py -i /root/setup/dev/advanced.cfg
+        # Retry data center deployment up to 3 times
+        for attempt in 1 2 3; do
+          echo "Data center deployment attempt $attempt/3"
+          if docker exec $(docker container ls --format=json -l | jq -r .ID) \
+            python /root/tools/marvin/marvin/deployDataCenter.py -i /root/setup/dev/advanced.cfg; then
+            echo "Data center deployment successful on attempt $attempt"
+            break
+          else
+            echo "Data center deployment failed on attempt $attempt"
+            if [ $attempt -eq 3 ]; then
+              echo "All data center deployment attempts failed"
+              exit 1
+            fi
+            echo "Waiting 30 seconds before retry..."
+            sleep 30
+          fi
+        done
 
         # Get the container ID of the running simulator
         CONTAINER_ID=$(docker ps --filter "ancestor=apache/cloudstack-simulator:${{ matrix.cloudstack-version }}" --format "{{.ID}}" | head -n1)

.github/workflows/acceptance.yml

@@ -47,6 +47,7 @@ jobs:
     name: Terraform ${{ matrix.terraform-version }} with Cloudstack ${{ matrix.cloudstack-version }}
     needs: [prepare-matrix]
     runs-on: ubuntu-latest
+    timeout-minutes: 90
     steps:
       - uses: actions/checkout@v4
       - name: Set up Go
@@ -86,6 +87,7 @@ jobs:
     name: OpenTofu ${{ matrix.opentofu-version }} with Cloudstack ${{ matrix.cloudstack-version }}
     needs: [prepare-matrix]
     runs-on: ubuntu-latest
+    timeout-minutes: 90
     steps:
       - uses: actions/checkout@v4
       - name: Set up Go

GNUmakefile

@@ -31,7 +31,7 @@ test: fmtcheck
 		xargs -t -n4 go test $(TESTARGS) -timeout=30s -parallel=4
 
 testacc: fmtcheck
-	TF_ACC=1 go test $(TEST) -v $(TESTARGS) -timeout 30m
+	TF_ACC=1 go test $(TEST) -v $(TESTARGS) -timeout 60m
 
 vet:
 	@echo "go vet ."