test test

Author: BartoszBlizniak

.github/workflows/acceptance-tests.yml

@@ -22,13 +22,32 @@ jobs:
 
     - uses: actions/checkout@v3
 
-    - name: Run tests
+    - name: Install test helper (gotestsum)
       run: |
-        TF_ACC=1 go test -v ./... -parallel=6 -timeout=30m
+        echo "Installing gotestsum for selective test reruns" 
+        go install gotest.tools/gotestsum@v1.12.0
+      env:
+        GOPATH: ${{ runner.temp }}/go
+      id: gotestsum
+
+    - name: Run acceptance tests with reruns
+      run: |
+        chmod +x scripts/run-acc-tests.sh
+        PATH="${{ runner.temp }}/go/bin:$PATH" TF_ACC=1 CLOUDSMITH_NAMESPACE=terraform-provider-testing \
+          CLOUDSMITH_API_KEY='${{ secrets.CLOUDSMITH_API_KEY }}' \
+          ./scripts/run-acc-tests.sh
       env:
         CLOUDSMITH_API_KEY: ${{ secrets.CLOUDSMITH_API_KEY }}
         CLOUDSMITH_NAMESPACE: terraform-provider-testing
 
+    - name: Upload test report
+      if: always()
+      uses: actions/upload-artifact@v4
+      with:
+        name: acceptance-test-report
+        path: test-report.xml
+        if-no-files-found: warn
+
     - name: Notify Slack on Success
       if: success()
       uses: rtCamp/action-slack-notify@v2.0.2

cloudsmith/resource_repository_upstream_test.go

@@ -133,7 +133,7 @@ resource "cloudsmith_repository" "test" {
 }
 
 resource "cloudsmith_repository_upstream" "ubuntu" {
-    distro_versions = ["ubuntu/trusty"]
+    distro_versions = ["ubuntu/trusty", "ubuntu/xenial"]
     namespace       = cloudsmith_repository.test.namespace
     repository      = cloudsmith_repository.test.slug
 	name            = cloudsmith_repository.test.name

scripts/run-acc-tests.sh

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+# Purpose: Run Terraform provider acceptance tests with automatic selective re-runs
+# of only the failing tests to mitigate flakiness (e.g. transient upstream/network issues).
+#
+# Environment variables (override as needed):
+#   PARALLEL           Go test -parallel value (default 6)
+#   TIMEOUT            Per test package timeout (default 30m)
+#   RERUN_MAX_TIMES    How many times to re-run a failing test (default 2)
+#   RERUN_FAILS        Max number of distinct failing tests eligible for rerun (default 50)
+#   EXTRA_ARGS         Extra args to pass through to `go test` after the rerun flags
+#   TF_ACC             Forced to 1 if unset (enables acceptance tests)
+#
+# Output:
+#   - test-report.xml  (JUnit format) for CI annotation
+#   - streaming stdout with concise first-run + rerun summaries
+#   - exit code 0 only if all tests eventually pass within rerun budget
+#
+# Requires: gotestsum (installed automatically if missing)
+set -euo pipefail
+
+PARALLEL=${PARALLEL:-6}
+TIMEOUT=${TIMEOUT:-30m}
+RERUN_MAX_TIMES=${RERUN_MAX_TIMES:-2}
+RERUN_FAILS=${RERUN_FAILS:-50}
+EXTRA_ARGS=${EXTRA_ARGS:-}
+
+export TF_ACC=${TF_ACC:-1}
+
+if ! command -v gotestsum >/dev/null 2>&1; then
+  echo "[INFO] Installing gotestsum (not found in PATH)" >&2
+  # Pin a version for reproducibility; update periodically.
+  go install gotest.tools/gotestsum@v1.12.0
+fi
+
+echo "[INFO] Running acceptance tests with selective reruns"
+echo "       parallel=${PARALLEL} timeout=${TIMEOUT} rerun_max_times=${RERUN_MAX_TIMES} rerun_fails=${RERUN_FAILS}" \
+     " extra_args='${EXTRA_ARGS}'"
+
+# We disable test caching (-count=1) to ensure fresh runs against live APIs.
+set +e
+gotestsum --format=short-verbose \
+  --junitfile test-report.xml \
+  --rerun-fails="${RERUN_FAILS}" \
+  --rerun-fails-max-times="${RERUN_MAX_TIMES}" \
+  --packages="./..." \
+  -- -count=1 -parallel="${PARALLEL}" -timeout="${TIMEOUT}" ${EXTRA_ARGS}
+status=$?
+set -e
+
+if [ $status -ne 0 ]; then
+  echo "[ERROR] Acceptance tests failed after selective reruns." >&2
+else
+  echo "[INFO] All acceptance tests passed (including reruns)." >&2
+fi
+
+exit $status