feat: replace LocalCluster with manual node spawning for e2e tests

- Replace LocalCluster with Port-based manual node spawning to fix OTP 28 compatibility
- Spawn nodes as separate OS processes using elixir --name
- Connect nodes via Node.connect and initialize Ra cluster manually
- Update all e2e tests to use new API (ports instead of cluster)
- Remove LocalCluster dependency from mix.exs
- Enable e2e tests in GitHub Actions workflow (no longer skipped)
- Update e2e README to reflect new approach
- Fix charlist warning in spawn_node function

This fixes the timeout issues with LocalCluster :peer module in OTP 28
while maintaining full multi-node testing capabilities.

Author: GSMLG-BOT

.github/workflows/e2e-test.yml

@@ -57,22 +57,10 @@ jobs:
       - name: Start EPMD (Erlang Port Mapper Daemon)
         run: epmd -daemon
 
-      - name: Skip E2E Distributed Tests (LocalCluster/OTP 28 incompatibility)
+      - name: Run E2E Distributed Tests
         run: |
-          echo "⚠️  Skipping e2e distributed tests due to known LocalCluster/OTP 28 issue"
-          echo ""
-          echo "Issue: LocalCluster 2.x uses :peer module which times out when spawning"
-          echo "       child nodes in OTP 28 (timeout after 15 seconds)"
-          echo ""
-          echo "Status: Fixed :not_alive error by using --name flag, but still get"
-          echo "        timeout in LocalCluster.start_link/2"
-          echo ""
-          echo "Workaround: Manual multi-node testing works (see e2e_test/README.md)"
-          echo ""
-          echo "Tracking: Upstream issue with :peer module in OTP 28"
-          echo ""
-          echo "✅ Marking as success (tests are skipped, not failing)"
-          exit 0
+          echo "Running e2e distributed tests with manual node spawning..."
+          MIX_ENV=e2e_test elixir --name test@127.0.0.1 --cookie concord_e2e_test -S mix test e2e_test/distributed/ --trace
         env:
           MIX_ENV: e2e_test
 

e2e_test/README.md

@@ -2,13 +2,11 @@
 
 This directory contains end-to-end tests for Concord that verify distributed behavior across multiple nodes in realistic scenarios.
 
-## ⚠️ Current Status: OTP 28 Compatibility Issue
+## ✅ Status: Working with Manual Node Spawning
 
-**LocalCluster 2.x is experiencing compatibility issues with Erlang/OTP 28.** The `:peer` module (used internally by LocalCluster) times out when attempting to start child nodes, preventing automated e2e tests from running.
+**E2E tests now use manual node spawning via Port instead of LocalCluster** to avoid OTP 28 compatibility issues. Tests spawn actual Erlang VM processes as separate OS processes and connect them via distributed Erlang.
 
-**Workaround:** Use manual multi-node testing (see "Manual Testing" section below) until this is resolved.
-
-**Tracking:** This is a known issue with the `:peer` module in OTP 28 that affects multiple distributed testing tools.
+**Approach:** Tests use `Port.open` to spawn `elixir --name` nodes, connect them with `Node.connect`, and initialize Ra cluster manually. This provides the same multi-node testing capabilities without the LocalCluster timeout issues.
 
 ## Overview
 

e2e_test/distributed/data_consistency_test.exs

@@ -6,13 +6,13 @@ defmodule Concord.E2E.DataConsistencyTest do
   @moduletag :distributed
 
   setup do
-    {:ok, nodes, cluster} = ClusterHelper.start_cluster(nodes: 3)
+    {:ok, nodes, ports} = ClusterHelper.start_cluster(nodes: 3)
 
     on_exit(fn ->
-      ClusterHelper.stop_cluster(cluster)
+      ClusterHelper.stop_cluster(ports)
     end)
 
-    %{nodes: nodes, cluster: cluster}
+    %{nodes: nodes, ports: ports}
   end
 
   describe "Data Consistency" do

e2e_test/distributed/leader_election_test.exs

@@ -7,13 +7,13 @@ defmodule Concord.E2E.LeaderElectionTest do
 
   setup do
     # Start a fresh cluster for each test
-    {:ok, nodes, cluster} = ClusterHelper.start_cluster(nodes: 3)
+    {:ok, nodes, ports} = ClusterHelper.start_cluster(nodes: 3)
 
     on_exit(fn ->
-      ClusterHelper.stop_cluster(cluster)
+      ClusterHelper.stop_cluster(ports)
     end)
 
-    %{nodes: nodes, cluster: cluster}
+    %{nodes: nodes, ports: ports}
   end
 
   describe "Leader Election" do

e2e_test/distributed/network_partition_test.exs

@@ -6,13 +6,13 @@ defmodule Concord.E2E.NetworkPartitionTest do
   @moduletag :distributed
 
   setup do
-    {:ok, nodes, cluster} = ClusterHelper.start_cluster(nodes: 5)
+    {:ok, nodes, ports} = ClusterHelper.start_cluster(nodes: 5)
 
     on_exit(fn ->
-      ClusterHelper.stop_cluster(cluster)
+      ClusterHelper.stop_cluster(ports)
     end)
 
-    %{nodes: nodes, cluster: cluster}
+    %{nodes: nodes, ports: ports}
   end
 
   describe "Network Partition" do

e2e_test/distributed/node_failure_test.exs

@@ -6,13 +6,13 @@ defmodule Concord.E2E.NodeFailureTest do
   @moduletag :distributed
 
   setup do
-    {:ok, nodes, cluster} = ClusterHelper.start_cluster(nodes: 3)
+    {:ok, nodes, ports} = ClusterHelper.start_cluster(nodes: 3)
 
     on_exit(fn ->
-      ClusterHelper.stop_cluster(cluster)
+      ClusterHelper.stop_cluster(ports)
     end)
 
-    %{nodes: nodes, cluster: cluster}
+    %{nodes: nodes, ports: ports}
   end
 
   describe "Node Failure Recovery" do

e2e_test/support/e2e_cluster_helper.ex

@@ -2,11 +2,11 @@ defmodule Concord.E2E.ClusterHelper do
   @moduledoc """
   Helper module for managing multi-node Concord clusters in e2e tests.
 
-  Uses LocalCluster to spawn actual Erlang nodes with full network isolation.
+  Uses manual node spawning via Port to avoid LocalCluster/OTP 28 compatibility issues.
   """
 
   @doc """
-  Starts a multi-node Concord cluster using LocalCluster.
+  Starts a multi-node Concord cluster by spawning separate Erlang VMs.
 
   ## Options
 
@@ -16,72 +16,93 @@ defmodule Concord.E2E.ClusterHelper do
 
   ## Returns
 
-    * `{:ok, nodes}` - List of started node names
+    * `{:ok, nodes, ports}` - List of started node names and their Port references
     * `{:error, reason}` - If cluster failed to start
 
   ## Examples
 
-      iex> {:ok, nodes} = E2E.ClusterHelper.start_cluster(nodes: 3)
+      iex> {:ok, nodes, ports} = E2E.ClusterHelper.start_cluster(nodes: 3)
       iex> length(nodes)
       3
   """
   def start_cluster(opts \\ []) do
     node_count = Keyword.get(opts, :nodes, 3)
     prefix = Keyword.get(opts, :prefix, "concord_e2e")
     wait_timeout = Keyword.get(opts, :wait_timeout, 30_000)
+    cookie = Keyword.get(opts, :cookie, :concord_e2e_test)
 
     IO.puts("Starting #{node_count}-node cluster with prefix '#{prefix}'...")
 
-    # Start cluster with LocalCluster 2.x API
-    # Use empty applications list to prevent auto-start of all apps
-    {:ok, cluster} =
-      LocalCluster.start_link(node_count,
-        prefix: String.to_atom(prefix),
-        applications: []
-      )
+    # Ensure current node is alive (required for distributed Erlang)
+    ensure_alive()
+
+    # Set cookie for cluster communication
+    Node.set_cookie(cookie)
+
+    # Start nodes as separate OS processes
+    nodes_and_ports =
+      Enum.map(1..node_count, fn i ->
+        node_name = :"#{prefix}#{i}@127.0.0.1"
+        port = spawn_node(node_name, cookie)
+        {node_name, port}
+      end)
 
-    # Get the node names
-    {:ok, nodes} = LocalCluster.nodes(cluster)
+    nodes = Enum.map(nodes_and_ports, fn {node, _port} -> node end)
+    ports = Enum.map(nodes_and_ports, fn {_node, port} -> port end)
 
     IO.puts("Started nodes: #{inspect(nodes)}")
 
+    # Wait for nodes to be reachable
+    Process.sleep(2000)
+
+    # Connect to all nodes
+    Enum.each(nodes, fn node ->
+      case Node.connect(node) do
+        true -> IO.puts("✓ Connected to #{node}")
+        false -> IO.puts("✗ Failed to connect to #{node}")
+        :ignored -> IO.puts("⚠ Already connected to #{node}")
+      end
+    end)
+
+    # Wait for connections to stabilize
+    Process.sleep(1000)
+
     # Initialize Concord on each node
     Enum.each(nodes, fn node ->
       IO.puts("Initializing Concord on #{node}...")
       :rpc.call(node, Application, :ensure_all_started, [:concord])
     end)
 
+    # Initialize Ra cluster on all nodes with full member list
+    initialize_ra_cluster(nodes)
+
     # Wait for cluster formation
     case wait_for_cluster_ready(nodes, wait_timeout) do
       :ok ->
         IO.puts("✓ Cluster ready with #{length(nodes)} nodes")
-        {:ok, nodes, cluster}
+        {:ok, nodes, ports}
 
       {:error, reason} ->
         IO.puts("✗ Cluster failed to start: #{inspect(reason)}")
-        LocalCluster.stop(cluster)
+        stop_cluster(ports)
         {:error, reason}
     end
   end
 
   @doc """
   Stops a running cluster and cleans up resources.
   """
-  def stop_cluster(cluster) do
-    {:ok, nodes} = LocalCluster.nodes(cluster)
-    IO.puts("Stopping cluster nodes: #{inspect(nodes)}")
+  def stop_cluster(ports) when is_list(ports) do
+    IO.puts("Stopping cluster nodes...")
 
-    # Stop Concord application on each node first
-    Enum.each(nodes, fn node ->
-      :rpc.call(node, Application, :stop, [:concord])
+    # Close all ports (kills the node processes)
+    Enum.each(ports, fn port ->
+      Port.close(port)
     end)
 
     # Give time for graceful shutdown
     Process.sleep(500)
 
-    # Stop the cluster
-    LocalCluster.stop(cluster)
-
     # Clean up data directories
     cleanup_data_dirs()
 
@@ -185,6 +206,140 @@ defmodule Concord.E2E.ClusterHelper do
 
   # Private functions
 
+  defp ensure_alive do
+    case Node.alive?() do
+      true ->
+        :ok
+
+      false ->
+        # Current node needs to be alive for distributed Erlang
+        # This should already be set by the test runner with --name flag
+        raise """
+        Current node is not alive! E2E tests require distributed Erlang.
+        Run tests with: elixir --name test@127.0.0.1 --cookie test -S mix test e2e_test/
+        """
+    end
+  end
+
+  defp spawn_node(node_name, cookie) do
+    # Get the current working directory and Mix environment
+    cwd = File.cwd!()
+    mix_env = "e2e_test"
+
+    # Build the command to start an iex node
+    # Use detached mode to run in background
+    cmd = "elixir"
+
+    args = [
+      "--name",
+      to_string(node_name),
+      "--cookie",
+      to_string(cookie),
+      "--no-halt",
+      "--erl",
+      "-kernel inet_dist_listen_min 9100 inet_dist_listen_max 9199",
+      "-S",
+      "mix",
+      "run",
+      "--no-start"
+    ]
+
+    # Spawn the node as a separate OS process
+    port =
+      Port.open(
+        {:spawn_executable, System.find_executable(cmd)},
+        [
+          :binary,
+          :exit_status,
+          {:args, args},
+          {:cd, cwd},
+          {:env, [{~c"MIX_ENV", String.to_charlist(mix_env)}]},
+          {:line, 1024}
+        ]
+      )
+
+    IO.puts("Spawned node #{node_name} via Port #{inspect(port)}")
+    port
+  end
+
+  defp initialize_ra_cluster(nodes) do
+    cluster_name = :concord_cluster
+    data_dir_base = "./data/e2e_test"
+
+    # Build server IDs for all nodes
+    server_ids = Enum.map(nodes, &{cluster_name, &1})
+
+    IO.puts("Initializing Ra cluster with members: #{inspect(server_ids)}")
+
+    # Initialize Ra cluster on each node
+    Enum.each(nodes, fn node ->
+      node_id = {cluster_name, node}
+      uid = node_id |> Tuple.to_list() |> Enum.join("_") |> String.replace("@", "_")
+      data_dir = "#{data_dir_base}/#{node}"
+
+      # Stop existing Ra server if running
+      case :rpc.call(node, :ra, :stop_server, [node_id]) do
+        :ok ->
+          IO.puts("✓ Stopped existing Ra server on #{node}")
+
+        {:error, :not_started} ->
+          IO.puts("⚠ Ra server was not started on #{node}")
+
+        {:error, :system_not_started} ->
+          IO.puts("⚠ Ra system not started on #{node}")
+
+        {:badrpc, _} = error ->
+          IO.puts("⚠ RPC error stopping Ra on #{node}: #{inspect(error)}")
+
+        error ->
+          IO.puts("⚠ Stop server on #{node} returned: #{inspect(error)}")
+      end
+
+      # Wait for shutdown
+      Process.sleep(500)
+
+      # Start Ra server with all members
+      server_config = %{
+        id: node_id,
+        uid: uid,
+        cluster_name: cluster_name,
+        machine: {:module, Concord.StateMachine, %{}},
+        log_init_args: %{
+          uid: uid,
+          data_dir: data_dir
+        },
+        initial_members: server_ids
+      }
+
+      case :rpc.call(node, :ra, :start_server, [server_config]) do
+        :ok ->
+          IO.puts("✓ Ra server started on #{node}")
+          :rpc.call(node, :ra, :trigger_election, [node_id])
+          :ok
+
+        {:ok, _} ->
+          IO.puts("✓ Ra server started on #{node}")
+          :ok
+
+        {:error, {:already_started, _}} ->
+          IO.puts("✓ Ra server already running on #{node}")
+          :ok
+
+        {:error, reason} ->
+          IO.puts("✗ Failed to start Ra server on #{node}: #{inspect(reason)}")
+          {:error, reason}
+
+        {:badrpc, reason} ->
+          IO.puts("✗ RPC error starting Ra on #{node}: #{inspect(reason)}")
+          {:error, {:badrpc, reason}}
+      end
+    end)
+
+    # Wait for Ra cluster to initialize
+    Process.sleep(3000)
+    :ok
+  end
+
   defp wait_for_cluster_ready(nodes, timeout) do
     start_time = System.monotonic_time(:millisecond)
     until = start_time + timeout

mix.exs

@@ -113,8 +113,7 @@ defmodule Concord.MixProject do
       {:opentelemetry_telemetry, "~> 1.1"},
       # Event streaming with GenStage
       {:gen_stage, "~> 1.2"},
-      # E2E testing
-      {:local_cluster, "~> 2.0", only: [:e2e_test], runtime: false},
+      # E2E testing (note: LocalCluster removed due to OTP 28 compatibility, using manual node spawning)
       {:httpoison, "~> 2.0", only: [:e2e_test], runtime: false},
       {:dialyxir, "~> 1.4", only: [:dev, :test], runtime: false},
       {:credo, "~> 1.7", only: [:dev, :test], runtime: false},