(ELI-466) more improvements and added concurrency to test deployments

TOEL2 · TOEL2 · commit 8922b8323d5a · 2025-11-04T16:24:40.000Z
diff --git a/.act/act_tests.mk b/.act/act_tests.mk
@@ -0,0 +1,39 @@
+ACT_IMAGE = ghcr.io/nhs-england-tools/github-runner-image:20230909-321fd1e-rt
+PREPROD_WORKFLOW  = .github/workflows/cicd-4-preprod-deploy.yaml
+JOB       = metadata
+
+# Usage: make act EVENT=.act/workflow_run_preprod.json TRIGGER_TYPE=workflow_run
+act-preprod-deploy:
+	@if [ -z "$(EVENT)" ]; then \
+		echo "Usage: make act EVENT=<path-to-event-json>"; \
+		exit 1; \
+	fi
+	@echo "Running act with event file: $(EVENT)"
+	ACT=true act \
+		-W $(PREPROD_WORKFLOW) \
+		--job $(JOB) \
+		--eventpath $(EVENT) \
+		-P ubuntu-latest=$(ACT_IMAGE) \
+		-s GITHUB_TOKEN="$$GITHUB_TOKEN" \
+		-s GH_TOKEN="$$GITHUB_TOKEN" \
+		--env GITHUB_REPOSITORY="$$REPO" \
+		--env PREPROD_WORKFLOW_ID=182365668 \
+		--env GITHUB_EVENT_NAME=$(TRIGGER_TYPE)
+
+
+#act-dev-deploy:
+#	@if [ -z "$(EVENT)" ]; then \
+#		echo "Usage: make act EVENT=<path-to-event-json>"; \
+#		exit 1; \
+#	fi
+#	@echo "Running act with event file: $(EVENT)"
+#	ACT=true act \
+#		-W $(WORKFLOW) \
+#		--job $(JOB) \
+#		--eventpath $(EVENT) \
+#		-P ubuntu-latest=$(ACT_IMAGE) \
+#		-s GITHUB_TOKEN="$$GITHUB_TOKEN" \
+#		-s GH_TOKEN="$$GITHUB_TOKEN" \
+#		--env GITHUB_REPOSITORY="$$REPO" \
+#		--env DEV_WORKFLOW_ID=143714547 \
+#		--env GITHUB_EVENT_NAME=$(TRIGGER_TYPE)
diff --git a/.act/auto_preprod_trigger_latest.json b/.act/auto_preprod_trigger_latest.json
@@ -0,0 +1,22 @@
+{
+  "event_name": "workflow_run",
+  "action": "completed",
+  "workflow": {
+    "name": "Test stage",
+    "path": ".github/workflows/cicd-3-test-deploy.yaml"
+  },
+  "workflow_run": {
+    "id": 18556637650,
+    "name": "Test stage",
+    "event": "push",
+    "status": "completed",
+    "conclusion": "success",
+    "head_branch": "main",
+    "head_sha": "501e5c4c311a765800af527021ed4c6a707c3474"
+  },
+  "repository": {
+    "full_name": "NHSDigital/eligibility-signposting-api",
+    "name": "eligibility-signposting-api",
+    "owner": {"login": "NHSDigital"}
+  }
+}
diff --git a/.act/auto_preprod_trigger_stale.json b/.act/auto_preprod_trigger_stale.json
@@ -0,0 +1,22 @@
+{
+  "event_name": "workflow_run",
+  "action": "completed",
+  "workflow": {
+    "name": "Test stage",
+    "path": ".github/workflows/cicd-3-test-deploy.yaml"
+  },
+  "workflow_run": {
+    "id": 18556637650,
+    "name": "Test stage",
+    "event": "push",
+    "status": "completed",
+    "conclusion": "success",
+    "head_branch": "main",
+    "head_sha": "b2675604f38a7c89f3cfe66f8b927b39d7ddedf6"
+  },
+  "repository": {
+    "full_name": "NHSDigital/eligibility-signposting-api",
+    "name": "eligibility-signposting-api",
+    "owner": {"login": "NHSDigital"}
+  }
+}
diff --git a/.act/manual_preprod_trigger.json b/.act/manual_preprod_trigger.json
@@ -0,0 +1,17 @@
+{
+  "event_name": "workflow_dispatch",
+  "ref": "refs/heads/main",
+  "inputs": {
+    "ref": "dev-20251015111601",
+    "release_type": "minor",
+    "reason": "manual promotion from test to pre-prod"
+  },
+  "repository": {
+    "full_name": "NHSDigital/eligibility-signposting-api",
+    "name": "eligibility-signposting-api",
+    "owner": { "login": "NHSDigital" }
+  },
+  "sender": {
+    "login": "tome"
+  }
+}
diff --git a/.github/workflows/cicd-3-test-deploy.yaml b/.github/workflows/cicd-3-test-deploy.yaml
@@ -5,6 +5,10 @@ on:
     workflows: ["2. CD | Deploy to Dev"]
     types: [completed]
 
+concurrency:
+  group: test-deployments
+  cancel-in-progress: false
+
 permissions:
   contents: read
   id-token: write
@@ -52,11 +56,6 @@ jobs:
       id-token: write
       contents: read
     steps:
-      - name: "Acquire deploy lock"
-        uses: softprops/turnstyle@v3
-        with:
-          poll-interval-seconds: 10
-
       - name: "Checkout same commit"
         uses: actions/checkout@v5
         with:
diff --git a/.github/workflows/cicd-4-preprod-deploy.yaml b/.github/workflows/cicd-4-preprod-deploy.yaml
@@ -17,12 +17,6 @@ on:
         description: "rc|patch|minor|major"
         required: true
         default: "rc"
-      allow_older:
-        description: "Allow deploying older than latest tested on main?"
-        required: false
-        type: choice
-        options: ["false","true"]
-        default: "false"
       reason:
         description: "Why are you doing a manual deployment?"
         required: true
@@ -42,37 +36,35 @@ jobs:
       latest_sha:   ${{ steps.resolver.outputs.latest_test_sha }}
       release_type: ${{ steps.release_type.outputs.release_type }}
 
+    env:
+      PREPROD_WORKFLOW_ID: "182365668"
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
     steps:
       - name: Checkout (full history & tags)
         uses: actions/checkout@v4
         with: { fetch-depth: 0 }
 
-      - name: Announce candidate ref
-        id: announce
+      - name: Force HTTPS remote for act
+        if: env.ACT == 'true'
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         shell: bash
         run: |
           set -euo pipefail
-          git fetch --tags --force --quiet
+          # Mask the token in logs (defense in depth)
+          echo "::add-mask::${GITHUB_TOKEN}"
+          # Option 1: rewrite just 'origin' to tokenized HTTPS
+          git remote set-url origin "https://x-access-token:${GITHUB_TOKEN}@github.com/${{ github.repository }}.git"
+          # Sanity check (will fail if token is missing/insufficient)
+          git ls-remote --tags origin >/dev/null
 
-          if [[ "${{ github.event_name }}" == "workflow_run" ]]; then
-            HEAD="${{ github.event.workflow_run.head_sha }}"
-            TAG="$(git tag --points-at "$HEAD" | grep '^dev-' | head -n1 || true)"
-          else
-            HEAD="$(git rev-list -n1 "${{ github.event.inputs.ref }}")"
-            TAG="${{ github.event.inputs.ref }}"
-          fi
-
-          echo "CANDIDATE_TAG=$TAG"
-          echo "CANDIDATE_SHA=$HEAD"
-
-          # Nice UI hints
-          echo "::notice title=PreProd candidate::dev tag: ${TAG} | sha: ${HEAD}"
-          {
-            echo "### PreProd candidate"
-            echo ""
-            echo "- dev tag: \`${TAG:-<none>}\`"
-            echo "- head sha: \`${HEAD:-<none>}\`"
-          } >> "$GITHUB_STEP_SUMMARY"
+      - name: Debug event
+        run: |
+          echo "GITHUB_EVENT_NAME=${GITHUB_EVENT_NAME}"
+          echo "Payload:" && cat "$GITHUB_EVENT_PATH" || true
 
       - name: Resolve THIS vs LATEST TEST + stale guard (auto only)
         id: resolver
@@ -81,11 +73,10 @@ jobs:
           EVENT_NAME: ${{ github.event_name }}
           WORKFLOW_RUN_HEAD_SHA: ${{ github.event.workflow_run.head_sha }}
           MANUAL_REF: ${{ github.event.inputs.ref }}
-          ALLOW_OLDER: ${{ github.event.inputs.allow_older }}
           WORKFLOW_NAME: "3. CD | Deploy to Test"
           BRANCH: "main"
           LIMIT: "30"
-        run: python3 scripts/pre-release_resolver.py
+        run: python3 scripts/workflow/pre-release_resolver.py
 
       - name: Resolve release_type (labels → default rc)
         id: release_type
@@ -96,7 +87,7 @@ jobs:
           THIS_SHA: ${{ steps.resolver.outputs.this_sha }}
           LATEST_TEST_SHA: ${{ steps.resolver.outputs.latest_test_sha }}
           MANUAL_RELEASE_TYPE: ${{ github.event.inputs.release_type }}
-        run: python3 scripts/release_type_resolver.py
+        run: python3 scripts/workflow/release_type_resolver.py
 
   deploy:
     name: "Call base-deploy.yml (PreProd)"
diff --git a/scripts/workflow/ci_utils.py b/scripts/workflow/ci_utils.py
@@ -10,17 +10,22 @@
 import os
 import subprocess
 import sys
-from typing import List, Optional, NoReturn
+from typing import List, Optional, NoReturn, Any
 
 BRANCH = os.getenv("BRANCH", "main")
+REPO_FALLBACK = "NHSDigital/eligibility-signposting-api"
 
 def fail(msg: str) -> NoReturn:
     print(f"::error::{msg}", file=sys.stderr)
     sys.exit(1)
 
-def run(cmd: List[str], *, check: bool = True) -> subprocess.CompletedProcess:
-    # cp = completed process (will use this to refer)
-    return subprocess.run(cmd, check=check, capture_output=True, text=True)
+def run(cmd: List[str], check: bool = True, **kwargs) -> subprocess.CompletedProcess:
+    cp = subprocess.run(cmd, check=False, capture_output=True, text=True, **kwargs)
+    if check and cp.returncode != 0:
+        raise subprocess.CalledProcessError(
+            cp.returncode, cmd, output=cp.stdout, stderr=cp.stderr
+        )
+    return cp
 
 def ensure_token() -> None:
     if not (os.getenv("GH_TOKEN") or os.getenv("GITHUB_TOKEN")):
@@ -36,10 +41,22 @@ def git_ok(args: List[str]) -> bool:
 def is_ancestor(older: str, newer: str) -> bool:
     return subprocess.run(["git", "merge-base", "--is-ancestor", older, newer]).returncode == 0
 
-def gh_json(args: List[str]) -> list:
-    cp = run(["gh", *args])
-    raw = cp.stdout.strip()
-    return json.loads(raw) if raw else []
+def gh_json(args: List[str]) -> Any:
+    # Map GITHUB_TOKEN -> GH_TOKEN (gh prefers GH_TOKEN)
+    if "GH_TOKEN" not in os.environ and os.environ.get("GITHUB_TOKEN"):
+        os.environ["GH_TOKEN"] = os.environ["GITHUB_TOKEN"]
+
+    # Ensure repo is explicit (act containers often need this)
+    repo = os.environ.get("GITHUB_REPOSITORY") or REPO_FALLBACK
+    base = ["gh", *args]
+    if "--repo" not in args and "-R" not in args:
+        base.extend(["--repo", repo])
+
+    cp = run(base, check=True)
+    try:
+        return json.loads(cp.stdout or "null")
+    except json.JSONDecodeError as e:
+        raise RuntimeError(f"Failed to parse gh JSON: {e}\nSTDOUT:\n{cp.stdout}\nSTDERR:\n{cp.stderr}")
 
 def gh_api(path: str, jq: Optional[str] = None) -> List[str]:
     """
diff --git a/scripts/workflow/pre-release_resolver.py b/scripts/workflow/pre-release_resolver.py
diff --git a/scripts/workflow/release_type_resolver.py b/scripts/workflow/release_type_resolver.py
