Adding wait & refactor the build check build param

tools/cloud-build/check_running_build.sh

@@ -15,18 +15,103 @@
 
 set -e
 
-TRIGGER_BUILD_CONFIG_PATH="$1"
+# Defaults
+CONFIG_PATH=""
+OUTPUT_FILE="${OUTPUT_FILE:-/workspace/sequential_test_env.sh}"
+SLEEP_INTERVAL=120
+MAX_WAIT_SECONDS=7200 # 2 hours default
 
-echo "$TRIGGER_BUILD_CONFIG_PATH"
+# Parse arguments
+while [[ "$#" -gt 0 ]]; do
+    case $1 in
+        --config) CONFIG_PATH="$2"; shift 2 ;;
+        *) TRIGGER_BUILD_CONFIG_PATH="$1"; shift ;;
+    esac
+done
 
-MATCHING_BUILDS=$(gcloud builds list --ongoing --format 'value(id)' --filter="substitutions.TRIGGER_BUILD_CONFIG_PATH=\"$TRIGGER_BUILD_CONFIG_PATH\"")
-MATCHING_COUNT=$(echo "$MATCHING_BUILDS" | wc -w)
+if [ -z "$TRIGGER_BUILD_CONFIG_PATH" ]; then
+    echo "No config path provided as argument. Attempting to fetch from build substitutions..."
+    if [ -n "$BUILD_ID" ]; then
+         TRIGGER_BUILD_CONFIG_PATH=$(gcloud builds describe "$BUILD_ID" --format='value(substitutions.TRIGGER_BUILD_CONFIG_PATH)' 2>/dev/null || true)
+         if [ -n "$TRIGGER_BUILD_CONFIG_PATH" ]; then
+              echo "Discovered config path from build: $TRIGGER_BUILD_CONFIG_PATH"
+         else
+              echo "Failed to retrieve TRIGGER_BUILD_CONFIG_PATH from substitutions."
+              exit 1
+         fi
+    else
+         echo "BUILD_ID is missing. Cannot fetch substitution."
+         exit 1
+    fi
+fi
+
+echo "Trigger Build Config Path: $TRIGGER_BUILD_CONFIG_PATH"
+
+# Get current build ID to exclude
+CURRENT_BUILD_ID=$BUILD_ID
+echo "Current Build ID: $CURRENT_BUILD_ID"
+
+check_builds() {
+    # Filter for ongoing builds with same TRIGGER_BUILD_CONFIG_PATH
+    # Exclude CURRENT_BUILD_ID
+    ALL_MATCHING=$(gcloud builds list --ongoing --format 'value(id)' --filter="substitutions.TRIGGER_BUILD_CONFIG_PATH=\"$TRIGGER_BUILD_CONFIG_PATH\"")
+
+    # Filter out current build ID
+    OTHER_MATCHING=""
+    for id in $ALL_MATCHING; do
+        if [ "$id" != "$CURRENT_BUILD_ID" ]; then
+            OTHER_MATCHING="$OTHER_MATCHING $id"
+        fi
+    done
+
+    echo "$OTHER_MATCHING"
+}
+
+WAIT_START=$(date +%s)
+
+while true; do
+    OTHER_BUILDS=$(check_builds)
+    OTHER_COUNT=$(echo "$OTHER_BUILDS" | wc -w)
+
+    if [ "$OTHER_COUNT" -eq 0 ]; then
+        echo "No other concurrent builds found. Proceeding."
+        break
+    fi
+
+
+    NOW=$(date +%s)
+    ELAPSED=$((NOW - WAIT_START))
+    if [ "$ELAPSED" -gt "$MAX_WAIT_SECONDS" ]; then
+        echo "Timeout reached while waiting for other builds to finish ($MAX_WAIT_SECONDS seconds)."
+        exit 1
+    fi
+
+    echo "Other build(s) running: $OTHER_BUILDS. Waiting $SLEEP_INTERVAL seconds... (Elapsed: ${ELAPSED}s)"
+    sleep "$SLEEP_INTERVAL"
+done
 
-if [ "$MATCHING_COUNT" -gt 1 ]; then
-	echo "Found more than 1 matching running build(s):"
-	echo "$MATCHING_BUILDS"
-	exit 1
+# Handle config.yml and network name
+if [ -n "$CONFIG_PATH" ]; then
+    if [ ! -f "$CONFIG_PATH" ]; then
+        echo "Config file not found: $CONFIG_PATH"
+        exit 1
+    fi
+
+    TEST_NAME=$(grep -E "^test_name:" "$CONFIG_PATH" | head -n 1 | awk '{print $2}' | tr -d '"' | tr -d "'")
+    if [ -z "$TEST_NAME" ]; then
+        echo "Could not extract test_name from $CONFIG_PATH"
+        exit 1
+    fi
+
+    # Generate a safe network name (convert to lower case, replace invalid chars with hyphens)
+    # Valid: lowercase letters, numbers, hyphens. Max 63 chars.
+    SAFE_NETWORK_NAME=$(echo "$TEST_NAME" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9-]/-/g' | sed 's/^-//' | sed 's/-$//' | cut -c 1-63)
+
+    echo "Test Name: $TEST_NAME"
+    echo "Generated Network Name: $SAFE_NETWORK_NAME"
+
+    echo "export NETWORK_NAME=\"$SAFE_NETWORK_NAME\"" > "$OUTPUT_FILE"
+    echo "Environment variables written to $OUTPUT_FILE"
 fi
 
-echo "No other matching running builds found (or only one)."
 exit 0

tools/cloud-build/daily-tests/builds/ansible-vm.yaml

@@ -28,7 +28,9 @@ steps:
 # While using static network names we are guarding against more than 1 instance running at a time (for multi-group tests)
 - id: check_for_running_build
   name: gcr.io/cloud-builders/gcloud
-  script: "tools/cloud-build/check_running_build.sh tools/cloud-build/daily-tests/builds/ansible-vm.yaml"
+  env:
+  - "BUILD_ID=$BUILD_ID"
+  script: "tools/cloud-build/check_running_build.sh ${_TRIGGER_BUILD_CONFIG_PATH}"
 
 - id: ansible-vm
   name: us-central1-docker.pkg.dev/$PROJECT_ID/hpc-toolkit-repo/test-runner

tools/cloud-build/daily-tests/builds/batch-mpi.yaml

@@ -40,7 +40,9 @@ steps:
 # While using static network names we are guarding against more than 1 instance running at a time (for multi-group tests)
 - id: check_for_running_build
   name: gcr.io/cloud-builders/gcloud
-  script: "tools/cloud-build/check_running_build.sh tools/cloud-build/daily-tests/builds/batch-mpi.yaml"
+  env:
+  - "BUILD_ID=$BUILD_ID"
+  script: "tools/cloud-build/check_running_build.sh ${_TRIGGER_BUILD_CONFIG_PATH}"
 
 # Test Batch MPI
 - id: batch-mpi

tools/cloud-build/daily-tests/builds/chrome-remote-desktop-ubuntu.yaml

@@ -27,7 +27,9 @@ steps:
 # While using static network names we are guarding against more than 1 instance running at a time (for multi-group tests)
 - id: check_for_running_build
   name: gcr.io/cloud-builders/gcloud
-  script: "tools/cloud-build/check_running_build.sh tools/cloud-build/daily-tests/builds/chrome-remote-desktop-ubuntu.yaml"
+  env:
+  - "BUILD_ID=$BUILD_ID"
+  script: "tools/cloud-build/check_running_build.sh ${_TRIGGER_BUILD_CONFIG_PATH}"
 
 - id: chrome-remote-desktop-ubuntu
   name: us-central1-docker.pkg.dev/$PROJECT_ID/hpc-toolkit-repo/test-runner

tools/cloud-build/daily-tests/builds/chrome-remote-desktop.yaml

@@ -27,7 +27,9 @@ steps:
 # While using static network names we are guarding against more than 1 instance running at a time (for multi-group tests)
 - id: check_for_running_build
   name: gcr.io/cloud-builders/gcloud
-  script: "tools/cloud-build/check_running_build.sh tools/cloud-build/daily-tests/builds/chrome-remote-desktop.yaml"
+  env:
+  - "BUILD_ID=$BUILD_ID"
+  script: "tools/cloud-build/check_running_build.sh ${_TRIGGER_BUILD_CONFIG_PATH}"
 
 # Test chrome-remote-desktop module
 - id: chrome-remote-desktop

tools/cloud-build/daily-tests/builds/cloud-batch.yaml

@@ -28,7 +28,9 @@ steps:
 # While using static network names we are guarding against more than 1 instance running at a time (for multi-group tests)
 - id: check_for_running_build
   name: gcr.io/cloud-builders/gcloud
-  script: "tools/cloud-build/check_running_build.sh tools/cloud-build/daily-tests/builds/cloud-batch.yaml"
+  env:
+  - "BUILD_ID=$BUILD_ID"
+  script: "tools/cloud-build/check_running_build.sh ${_TRIGGER_BUILD_CONFIG_PATH}"
 
 # Test Cloud Batch Example
 - id: cloud-batch

tools/cloud-build/daily-tests/builds/e2e.yaml

@@ -26,7 +26,9 @@ steps:
 # While using static network names we are guarding against more than 1 instance running at a time (for multi-group tests)
 - id: check_for_running_build
   name: gcr.io/cloud-builders/gcloud
-  script: "tools/cloud-build/check_running_build.sh tools/cloud-build/daily-tests/builds/e2e.yaml"
+  env:
+  - "BUILD_ID=$BUILD_ID"
+  script: "tools/cloud-build/check_running_build.sh ${_TRIGGER_BUILD_CONFIG_PATH}"
 
 - name: us-central1-docker.pkg.dev/$PROJECT_ID/hpc-toolkit-repo/test-runner
   automapSubstitutions: true

tools/cloud-build/daily-tests/builds/gcluster-dockerfile.yaml

@@ -22,7 +22,9 @@ timeout: 3600s  # 1hr
 steps:
 - id: check_for_running_build
   name: gcr.io/cloud-builders/gcloud
-  script: "tools/cloud-build/check_running_build.sh tools/cloud-build/daily-tests/builds/gcluster-dockerfile.yaml"
+  env:
+  - "BUILD_ID=$BUILD_ID"
+  script: "tools/cloud-build/check_running_build.sh ${_TRIGGER_BUILD_CONFIG_PATH}"
 
 # uses a pre-built Google Cloud image containing Docker CLI tool.
 - id: build-docker-image

tools/cloud-build/daily-tests/builds/gke-a2-highgpu-kueue-onspot.yaml

@@ -30,7 +30,9 @@ steps:
 # While using static network names we are guarding against more than 1 instance running at a time (for multi-group tests)
 - id: check_for_running_build
   name: gcr.io/cloud-builders/gcloud
-  script: "tools/cloud-build/check_running_build.sh tools/cloud-build/daily-tests/builds/gke-a2-highgpu-kueue-onspot.yaml"
+  env:
+  - "BUILD_ID=$BUILD_ID"
+  script: "tools/cloud-build/check_running_build.sh ${_TRIGGER_BUILD_CONFIG_PATH}"
 
 - id: gke-a2-highgpu-kueue-onspot-test
   name: us-central1-docker.pkg.dev/$PROJECT_ID/hpc-toolkit-repo/test-runner

tools/cloud-build/daily-tests/builds/gke-a2-highgpu-kueue.yaml

@@ -30,7 +30,9 @@ steps:
 # While using static network names we are guarding against more than 1 instance running at a time (for multi-group tests)
 - id: check_for_running_build
   name: gcr.io/cloud-builders/gcloud
-  script: "tools/cloud-build/check_running_build.sh tools/cloud-build/daily-tests/builds/gke-a2-highgpu-kueue.yaml"
+  env:
+  - "BUILD_ID=$BUILD_ID"
+  script: "tools/cloud-build/check_running_build.sh ${_TRIGGER_BUILD_CONFIG_PATH}"
 
 - id: gke-a2-highgpu-kueue-test
   name: us-central1-docker.pkg.dev/$PROJECT_ID/hpc-toolkit-repo/test-runner

tools/cloud-build/daily-tests/builds/gke-a3-highgpu-onspot.yaml

@@ -30,7 +30,9 @@ steps:
 # While using static network names we are guarding against more than 1 instance running at a time (for multi-group tests)
 - id: check_for_running_build
   name: gcr.io/cloud-builders/gcloud
-  script: "tools/cloud-build/check_running_build.sh tools/cloud-build/daily-tests/builds/gke-a3-highgpu-onspot.yaml"
+  env:
+  - "BUILD_ID=$BUILD_ID"
+  script: "tools/cloud-build/check_running_build.sh ${_TRIGGER_BUILD_CONFIG_PATH}"
 
 - id: gke-a3-highgpu-onspot
   name: us-central1-docker.pkg.dev/$PROJECT_ID/hpc-toolkit-repo/test-runner

tools/cloud-build/daily-tests/builds/gke-a3-highgpu.yaml

@@ -30,7 +30,9 @@ steps:
 # While using static network names we are guarding against more than 1 instance running at a time (for multi-group tests)
 - id: check_for_running_build
   name: gcr.io/cloud-builders/gcloud
-  script: "tools/cloud-build/check_running_build.sh tools/cloud-build/daily-tests/builds/gke-a3-highgpu.yaml"
+  env:
+  - "BUILD_ID=$BUILD_ID"
+  script: "tools/cloud-build/check_running_build.sh ${_TRIGGER_BUILD_CONFIG_PATH}"
 
 - id: gke-a3-highgpu
   name: us-central1-docker.pkg.dev/$PROJECT_ID/hpc-toolkit-repo/test-runner

tools/cloud-build/daily-tests/builds/gke-a3-megagpu-onspot.yaml

@@ -30,7 +30,9 @@ steps:
 # While using static network names we are guarding against more than 1 instance running at a time (for multi-group tests)
 - id: check_for_running_build
   name: gcr.io/cloud-builders/gcloud
-  script: "tools/cloud-build/check_running_build.sh tools/cloud-build/daily-tests/builds/gke-a3-megagpu-onspot.yaml"
+  env:
+  - "BUILD_ID=$BUILD_ID"
+  script: "tools/cloud-build/check_running_build.sh ${_TRIGGER_BUILD_CONFIG_PATH}"
 
 - id: gke-a3-megagpu-onspot
   name: us-central1-docker.pkg.dev/$PROJECT_ID/hpc-toolkit-repo/test-runner

tools/cloud-build/daily-tests/builds/gke-a3-megagpu.yaml

@@ -30,7 +30,9 @@ steps:
 # While using static network names we are guarding against more than 1 instance running at a time (for multi-group tests)
 - id: check_for_running_build
   name: gcr.io/cloud-builders/gcloud
-  script: "tools/cloud-build/check_running_build.sh tools/cloud-build/daily-tests/builds/gke-a3-megagpu.yaml"
+  env:
+  - "BUILD_ID=$BUILD_ID"
+  script: "tools/cloud-build/check_running_build.sh ${_TRIGGER_BUILD_CONFIG_PATH}"
 
 - id: gke-a3-megagpu
   name: us-central1-docker.pkg.dev/$PROJECT_ID/hpc-toolkit-repo/test-runner

tools/cloud-build/daily-tests/builds/gke-a3-ultragpu-onspot.yaml

@@ -34,7 +34,9 @@ steps:
 # While using static network names we are guarding against more than 1 instance running at a time (for multi-group tests)
 - id: check_for_running_build
   name: gcr.io/cloud-builders/gcloud
-  script: "tools/cloud-build/check_running_build.sh tools/cloud-build/daily-tests/builds/gke-a3-ultragpu-onspot.yaml"
+  env:
+  - "BUILD_ID=$BUILD_ID"
+  script: "tools/cloud-build/check_running_build.sh ${_TRIGGER_BUILD_CONFIG_PATH}"
 
 - id: gke-a3-ultragpu-onspot
   name: us-central1-docker.pkg.dev/$PROJECT_ID/hpc-toolkit-repo/test-runner

tools/cloud-build/daily-tests/builds/gke-a3-ultragpu.yaml

@@ -34,7 +34,9 @@ steps:
 # While using static network names we are guarding against more than 1 instance running at a time (for multi-group tests)
 - id: check_for_running_build
   name: gcr.io/cloud-builders/gcloud
-  script: "tools/cloud-build/check_running_build.sh tools/cloud-build/daily-tests/builds/gke-a3-ultragpu.yaml"
+  env:
+  - "BUILD_ID=$BUILD_ID"
+  script: "tools/cloud-build/check_running_build.sh ${_TRIGGER_BUILD_CONFIG_PATH}"
 
 - id: gke-a3-ultragpu
   name: us-central1-docker.pkg.dev/$PROJECT_ID/hpc-toolkit-repo/test-runner

tools/cloud-build/daily-tests/builds/gke-a4-onspot.yaml

@@ -34,7 +34,9 @@ steps:
 # While using static network names we are guarding against more than 1 instance running at a time (for multi-group tests)
 - id: check_for_running_build
   name: gcr.io/cloud-builders/gcloud
-  script: "tools/cloud-build/check_running_build.sh tools/cloud-build/daily-tests/builds/gke-a4-onspot.yaml"
+  env:
+  - "BUILD_ID=$BUILD_ID"
+  script: "tools/cloud-build/check_running_build.sh ${_TRIGGER_BUILD_CONFIG_PATH}"
 
 - id: gke-a4-onspot
   name: us-central1-docker.pkg.dev/$PROJECT_ID/hpc-toolkit-repo/test-runner

tools/cloud-build/daily-tests/builds/gke-a4.yaml

@@ -34,7 +34,9 @@ steps:
 # While using static network names we are guarding against more than 1 instance running at a time (for multi-group tests)
 - id: check_for_running_build
   name: gcr.io/cloud-builders/gcloud
-  script: "tools/cloud-build/check_running_build.sh tools/cloud-build/daily-tests/builds/gke-a4.yaml"
+  env:
+  - "BUILD_ID=$BUILD_ID"
+  script: "tools/cloud-build/check_running_build.sh ${_TRIGGER_BUILD_CONFIG_PATH}"
 
 - id: gke-a4
   name: us-central1-docker.pkg.dev/$PROJECT_ID/hpc-toolkit-repo/test-runner

tools/cloud-build/daily-tests/builds/gke-a4x.yaml

@@ -36,7 +36,9 @@ steps:
 # While using static network names we are guarding against more than 1 instance running at a time (for multi-group tests)
 - id: check_for_running_build
   name: gcr.io/cloud-builders/gcloud
-  script: "tools/cloud-build/check_running_build.sh tools/cloud-build/daily-tests/builds/gke-a4x.yaml"
+  env:
+  - "BUILD_ID=$BUILD_ID"
+  script: "tools/cloud-build/check_running_build.sh ${_TRIGGER_BUILD_CONFIG_PATH}"
 
 - id: gke-a4x
   name: us-central1-docker.pkg.dev/$PROJECT_ID/hpc-toolkit-repo/test-runner

tools/cloud-build/daily-tests/builds/gke-g4.yaml

@@ -30,7 +30,9 @@ steps:
 # While using static network names we are guarding against more than 1 instance running at a time (for multi-group tests)
 - id: check_for_running_build
   name: gcr.io/cloud-builders/gcloud
-  script: "tools/cloud-build/check_running_build.sh tools/cloud-build/daily-tests/builds/gke-g4.yaml"
+  env:
+  - "BUILD_ID=$BUILD_ID"
+  script: "tools/cloud-build/check_running_build.sh ${_TRIGGER_BUILD_CONFIG_PATH}"
 
 - id: gke-g4
   name: us-central1-docker.pkg.dev/$PROJECT_ID/hpc-toolkit-repo/test-runner

tools/cloud-build/daily-tests/builds/gke-h4d-onspot.yaml

@@ -33,7 +33,9 @@ steps:
 # While using static network names we are guarding against more than 1 instance running at a time (for multi-group tests)
 - id: check_for_running_build
   name: gcr.io/cloud-builders/gcloud
-  script: "tools/cloud-build/check_running_build.sh tools/cloud-build/daily-tests/builds/gke-h4d-onspot.yaml"
+  env:
+  - "BUILD_ID=$BUILD_ID"
+  script: "tools/cloud-build/check_running_build.sh ${_TRIGGER_BUILD_CONFIG_PATH}"
 
 - id: gke-h4d-onspot
   name: us-central1-docker.pkg.dev/$PROJECT_ID/hpc-toolkit-repo/test-runner

tools/cloud-build/daily-tests/builds/gke-h4d.yaml

@@ -33,7 +33,9 @@ steps:
 # While using static network names we are guarding against more than 1 instance running at a time (for multi-group tests)
 - id: check_for_running_build
   name: gcr.io/cloud-builders/gcloud
-  script: "tools/cloud-build/check_running_build.sh tools/cloud-build/daily-tests/builds/gke-h4d.yaml"
+  env:
+  - "BUILD_ID=$BUILD_ID"
+  script: "tools/cloud-build/check_running_build.sh ${_TRIGGER_BUILD_CONFIG_PATH}"
 
 - id: gke-h4d
   name: us-central1-docker.pkg.dev/$PROJECT_ID/hpc-toolkit-repo/test-runner

tools/cloud-build/daily-tests/builds/gke-inactive-reservation.yaml

@@ -30,7 +30,9 @@ steps:
 # While using static network names we are guarding against more than 1 instance running at a time (for multi-group tests)
 - id: check_for_running_build
   name: gcr.io/cloud-builders/gcloud
-  script: "tools/cloud-build/check_running_build.sh tools/cloud-build/daily-tests/builds/gke-inactive-reservation.yaml"
+  env:
+  - "BUILD_ID=$BUILD_ID"
+  script: "tools/cloud-build/check_running_build.sh ${_TRIGGER_BUILD_CONFIG_PATH}"
 
 - id: gke-inactive-reservation
   name: us-central1-docker.pkg.dev/$PROJECT_ID/hpc-toolkit-repo/test-runner

tools/cloud-build/daily-tests/builds/gke-managed-hyperdisk.yaml

@@ -31,7 +31,9 @@ steps:
 # While using static network names we are guarding against more than 1 instance running at a time (for multi-group tests)
 - id: check_for_running_build
   name: gcr.io/cloud-builders/gcloud
-  script: "tools/cloud-build/check_running_build.sh tools/cloud-build/daily-tests/builds/gke-managed-hyperdisk.yaml"
+  env:
+  - "BUILD_ID=$BUILD_ID"
+  script: "tools/cloud-build/check_running_build.sh ${_TRIGGER_BUILD_CONFIG_PATH}"
 
 # Test GKE
 - id: gke-managed-hyperdisk

tools/cloud-build/daily-tests/builds/gke-managed-lustre.yaml

@@ -33,7 +33,9 @@ steps:
 # While using static network names we are guarding against more than 1 instance running at a time (for multi-group tests)
 - id: check_for_running_build
   name: gcr.io/cloud-builders/gcloud
-  script: "tools/cloud-build/check_running_build.sh tools/cloud-build/daily-tests/builds/gke-managed-lustre.yaml"
+  env:
+  - "BUILD_ID=$BUILD_ID"
+  script: "tools/cloud-build/check_running_build.sh ${_TRIGGER_BUILD_CONFIG_PATH}"
 
 - id: gke-managed-lustre
   name: us-central1-docker.pkg.dev/$PROJECT_ID/hpc-toolkit-repo/test-runner

tools/cloud-build/daily-tests/builds/gke-storage.yaml

@@ -32,7 +32,9 @@ steps:
 # While using static network names we are guarding against more than 1 instance running at a time (for multi-group tests)
 - id: check_for_running_build
   name: gcr.io/cloud-builders/gcloud
-  script: "tools/cloud-build/check_running_build.sh tools/cloud-build/daily-tests/builds/gke-storage.yaml"
+  env:
+  - "BUILD_ID=$BUILD_ID"
+  script: "tools/cloud-build/check_running_build.sh ${_TRIGGER_BUILD_CONFIG_PATH}"
 
 ## Test GKE
 - id: gke-storage

tools/cloud-build/daily-tests/builds/gke-tpu-7x.yaml

@@ -31,7 +31,9 @@ steps:
 # While using static network names we are guarding against more than 1 instance running at a time (for multi-group tests)
 - id: check_for_running_build
   name: gcr.io/cloud-builders/gcloud
-  script: "tools/cloud-build/check_running_build.sh tools/cloud-build/daily-tests/builds/gke-tpu-7x.yaml"
+  env:
+  - "BUILD_ID=$BUILD_ID"
+  script: "tools/cloud-build/check_running_build.sh ${_TRIGGER_BUILD_CONFIG_PATH}"
 
 - id: gke-tpu-7x
   name: us-central1-docker.pkg.dev/$PROJECT_ID/hpc-toolkit-repo/test-runner