fix: improve e2e test cleanup and reliability

- Add cleanup function with trap for EXIT/INT/TERM signals in shell script
- Pre-compile before spawning nodes to avoid build lock contention
- Kill orphan concord_e2e processes at test start and exit
- Add --no-compile flag to spawned nodes
- Start telemetry apps on spawned nodes to avoid handler warnings
- Improve error messages when node is not distributed
- Add try/catch and pkill fallback in stop_cluster/1

Author: GSMLG-BOT

e2e_test/support/e2e_cluster_helper.ex

@@ -100,6 +100,10 @@ defmodule Concord.E2E.ClusterHelper do
 
       # Load Concord application code (but don't start it - we'll manually start Ra servers)
       :rpc.call(node, Application, :load, [:concord])
+
+      # Start telemetry apps to avoid handler warnings
+      :rpc.call(node, Application, :ensure_all_started, [:telemetry])
+      :rpc.call(node, Application, :ensure_all_started, [:telemetry_poller])
     end)
 
     # Initialize Ra cluster on all nodes with full member list
@@ -128,13 +132,20 @@ defmodule Concord.E2E.ClusterHelper do
     # Check if port is still open before closing to avoid errors
     Enum.each(ports, fn port ->
       if Port.info(port) != nil do
-        Port.close(port)
+        try do
+          Port.close(port)
+        catch
+          _, _ -> :ok
+        end
       end
     end)
 
     # Give time for graceful shutdown
     Process.sleep(500)
 
+    # Force kill any remaining processes
+    System.cmd("pkill", ["-9", "-f", "concord_e2e"], stderr_to_stdout: true)
+
     # Clean up data directories
     cleanup_data_dirs()
 
@@ -278,7 +289,8 @@ defmodule Concord.E2E.ClusterHelper do
       "-S",
       "mix",
       "run",
-      "--no-start"
+      "--no-start",
+      "--no-compile"
     ]
 
     # Spawn the node as a separate OS process

e2e_test/test_helper.exs

@@ -1,19 +1,39 @@
+# Clean up orphan processes from previous runs
+defmodule Concord.E2E.Cleanup do
+  def kill_orphans do
+    IO.puts("Cleaning up orphan e2e processes...")
+    System.cmd("pkill", ["-9", "-f", "concord_e2e"], stderr_to_stdout: true)
+    Process.sleep(500)
+  end
+end
+
+Concord.E2E.Cleanup.kill_orphans()
+
 # Ensure the test runner node is alive (distributed Erlang)
 # This should be handled by the mix alias passing --name flag
 case Node.alive?() do
   true ->
-    IO.puts("✓ Test runner node is alive: #{Node.self()}")
+    IO.puts("✓ Test runner node: #{Node.self()}")
     IO.puts("✓ Cookie: #{Node.get_cookie()}")
 
   false ->
     IO.puts("""
-    ✗ Error: Test runner node is not alive!
-
-    E2E tests require distributed Erlang to spawn cluster nodes.
-    The mix alias should handle this automatically.
-
-    If running manually, use:
-      MIX_ENV=e2e_test elixir --name test@127.0.0.1 --cookie test_cookie -S mix test e2e_test/
+    ╔════════════════════════════════════════════════════════════════════════════╗
+    ║                    ERROR: Test runner node is not distributed!             ║
+    ╠════════════════════════════════════════════════════════════════════════════╣
+    ║                                                                            ║
+    ║  E2E tests require distributed Erlang to spawn cluster nodes.              ║
+    ║                                                                            ║
+    ║  Use one of these methods:                                                 ║
+    ║                                                                            ║
+    ║    1. mix test.e2e (recommended)                                           ║
+    ║    2. ./scripts/run_e2e_tests.sh e2e_test/                                 ║
+    ║                                                                            ║
+    ║  Manual invocation:                                                        ║
+    ║    MIX_ENV=e2e_test elixir --name test@127.0.0.1 --cookie test_cookie \\   ║
+    ║      -S mix test e2e_test/                                                 ║
+    ║                                                                            ║
+    ╚════════════════════════════════════════════════════════════════════════════╝
     """)
 
     System.halt(1)
@@ -22,6 +42,12 @@ end
 # Start required applications
 {:ok, _} = Application.ensure_all_started(:telemetry)
 
+# Register cleanup on exit
+System.at_exit(fn _ ->
+  IO.puts("\nCleaning up e2e processes on exit...")
+  System.cmd("pkill", ["-9", "-f", "concord_e2e"], stderr_to_stdout: true)
+end)
+
 # Start ExUnit with specific configuration for e2e tests
 ExUnit.start(
   exclude: [:skip],

scripts/run_e2e_tests.sh

@@ -10,11 +10,29 @@ RED='\033[0;31m'
 YELLOW='\033[1;33m'
 NC='\033[0m' # No Color
 
-echo -e "${YELLOW}=== Starting E2E Test Suite ===${NC}"
-
 # Set MIX_ENV
 export MIX_ENV=e2e_test
 
+# Function to clean up orphan processes
+cleanup() {
+    echo -e "\n${YELLOW}Cleaning up e2e processes...${NC}"
+    pkill -9 -f "concord_e2e" 2>/dev/null || true
+    rm -rf ./data/e2e_test 2>/dev/null || true
+}
+
+# Trap to ensure cleanup on exit/interrupt
+trap cleanup EXIT INT TERM
+
+# Clean up orphans before starting
+cleanup
+sleep 1
+
+echo -e "${YELLOW}=== Starting E2E Test Suite ===${NC}"
+
+# Pre-compile to avoid build lock contention
+echo -e "${YELLOW}Pre-compiling...${NC}"
+mix compile --force
+
 # Parse arguments
 TEST_PATH="${1:-e2e_test/}"
 NODE_NAME="${NODE_NAME:-test@127.0.0.1}"