Refactor flakiness workflow and CI

Author: soulomoon

.github/workflows/flakiness.yml

@@ -66,9 +66,11 @@ jobs:
         run: |
           cabal --version
           ghc --version
-      - name: build
-        run: |
-          cabal build all
+      - name: Build
+        env:
+          PATTERN_FILE: 'scripts/flaky-test-patterns.txt'
+          RUN_MODE: 'build'
+        run: HLS_TEST_EXE="$(cabal exec which hls)" bash scripts/flaky-test-loop.sh
 
       - name: Run flakiness loop
         id: run-loop
@@ -81,6 +83,7 @@ jobs:
           TEST_PATTERNS: ${{ github.event.inputs.test_patterns }}
           PATTERN_FILE: 'scripts/flaky-test-patterns.txt'
           NO_BUILD_ONCE: '1'
+          RUN_MODE: 'run'
           # HLS_TEST_EXE: 'hls' # HLS_WRAPPER_TEST_EXE: 'hls-wrapper'
         run: |
           # Run with a sensible default of 500 iterations on PRs;

scripts/flaky-test-loop.sh

@@ -10,6 +10,7 @@
 #   SHOW_EVERY    : print progress/iteration header every N iterations (default: 100, 1 = every run, <=0 = disabled)
 #   LOG_STDERR    : set to 1 to enable verbose stderr logging (HLS_TEST_LOG_STDERR & HLS_TEST_HARNESS_STDERR) (default: 1)
 #   NO_BUILD_ONCE : set to non-empty to skip the initial cabal build step
+#   RUN_MODE      : choose 'build' (build once and exit), 'run' (skip upfront build and just run), or 'both' (default)
 #
 # Test selection:
 #   TEST_PATTERNS : comma-separated list of entries to run each iteration.
@@ -37,6 +38,7 @@ MAX_ITER="${MAX_ITER:-}"
 SLEEP_SECS="${SLEEP_SECS:-0}"
 SHOW_EVERY="${SHOW_EVERY:-1}"
 LOG_STDERR="${LOG_STDERR:-1}"
+RUN_MODE="${RUN_MODE:-both}"  # build | run | both
 
 # Allow providing a positional max iteration: ./open-close-loop.sh 50
 if [[ $# -ge 1 && -z "${MAX_ITER}" ]]; then
@@ -96,10 +98,11 @@ if [[ ${#items[@]} -eq 0 ]]; then
   exit 2
 fi
 
-# Build required test binaries once upfront (unless NO_BUILD_ONCE is set)
-if [[ -z "${NO_BUILD_ONCE:-}" ]]; then
-  # collect unique BIN names
+# Helper to build required test binaries once
+build_required_bins_once() {
+  # collect unique BIN names from global 'items'
   declare -a bins_to_build=()
+  local it bin seen b
   for it in "${items[@]}"; do
     bin="${it%%::*}"; seen=0
     if (( ${#bins_to_build[@]} > 0 )); then
@@ -110,11 +113,40 @@ if [[ -z "${NO_BUILD_ONCE:-}" ]]; then
   if (( ${#bins_to_build[@]} > 0 )); then
     echo "[loop] Building test targets once upfront: ${bins_to_build[*]}" >&2
     if ! cabal build "${bins_to_build[@]}" >&2; then
-      echo "[loop][error] Build failed. Cannot proceed with tests." >&2
-      exit 2
+      echo "[loop][error] Build failed." >&2
+      return 2
     fi
-    echo "[loop] Build succeeded. Proceeding with tests." >&2
+    echo "[loop] Build succeeded." >&2
   fi
+  return 0
+}
+
+# Honor RUN_MODE before any build/run
+case "${RUN_MODE}" in
+  build)
+    if ! build_required_bins_once; then exit 2; fi
+    echo "[loop] RUN_MODE=build completed. Exiting without running tests." >&2
+    exit 0
+    ;;
+  run)
+    echo "[loop] RUN_MODE=run: skipping upfront build, proceeding to run loop." >&2
+    ;;
+  both)
+    : # default behavior below
+    ;;
+  *)
+    echo "[loop][error] Invalid RUN_MODE='${RUN_MODE}'. Use one of: build | run | both." >&2
+    exit 2
+    ;;
+esac
+
+# Build required test binaries once upfront (unless NO_BUILD_ONCE is set or RUN_MODE=run)
+if [[ -z "${NO_BUILD_ONCE:-}" && "${RUN_MODE}" != "run" ]]; then
+  if ! build_required_bins_once; then
+    echo "[loop][error] Cannot proceed with tests due to build failure." >&2
+    exit 2
+  fi
+  echo "[loop] Proceeding with tests." >&2
 fi
 
 # Resolve binary path by name (cache results)

scripts/flaky-test-patterns.txt

@@ -1,7 +1,7 @@
 # One tasty pattern per line. Lines starting with # are comments.
 # Blank lines are ignored.
 
-# open close
+open close
 # non local variable
 # Notification Handlers
 # bidirectional module dependency with hs-boot
@@ -17,4 +17,4 @@
 # hls-hlint-plugin-tests::adding hlint flags to plugin configuration removes hlint diagnostics
 # hls-explicit-imports-plugin-tests::ExplicitUsualCase inlay hints with client caps
 # hls-class-plugin-tests::Creates a placeholder for fmap
-hls-rename-plugin-tests::Rename
+# hls-rename-plugin-tests::Rename