fix: change Connect and ReplicationConnection to be temporary (#1495)

We don't want them to be transient as a database connection might have
issues and the supervisor will keep trying to connect but if the
database is completely inaccessible the supervisor will reach max
restarts and causing the whole partition supervisor to crash.

Connect.lookup_or_start_connection ensures that the connection is back
up when needed so we don't need to worry about restarting theses
processes straightaway

Author: edgurgel

lib/realtime/tenants/connect.ex

@@ -7,7 +7,7 @@ defmodule Realtime.Tenants.Connect do
   * `:check_connect_region_interval` - The interval in milliseconds to check if this process is in the correct region. If the region is not correct it stops the connection.
   * `:erpc_timeout` - The timeout in milliseconds for the `:erpc` calls to the tenant's database.
   """
-  use GenServer, restart: :transient
+  use GenServer, restart: :temporary
 
   use Realtime.Logs
 

lib/realtime/tenants/replication_connection.ex

@@ -97,7 +97,7 @@ defmodule Realtime.Tenants.ReplicationConnection do
     child_spec = %{
       id: __MODULE__,
       start: {Wrapper, :start_link, [opts, init_timeout]},
-      restart: :transient,
+      restart: :temporary,
       type: :worker
     }
 

mix.exs

@@ -4,7 +4,7 @@ defmodule Realtime.MixProject do
   def project do
     [
       app: :realtime,
-      version: "2.42.0",
+      version: "2.42.1",
       elixir: "~> 1.17.3",
       elixirc_paths: elixirc_paths(Mix.env()),
       start_permanent: Mix.env() == :prod,

test/realtime/tenants/connect_test.exs

@@ -35,6 +35,21 @@ defmodule Realtime.Tenants.ConnectTest do
     refute_receive {:DOWN, ^ref, :process, ^pid, _reason}, timeout
   end
 
+  describe "temporary process" do
+    test "starts a temporary process", %{tenant: tenant} do
+      assert {:ok, _} = Connect.lookup_or_start_connection(tenant.external_id)
+      pid = Connect.whereis(tenant.external_id)
+      # Brutally kill the process
+      Process.exit(pid, :kill)
+      assert_process_down(pid)
+      # Wait to ensure that the process has not restarted
+      Process.sleep(1000)
+
+      # Temporary process should not be registered in syn
+      refute Connect.whereis(tenant.external_id)
+    end
+  end
+
   describe "handle cold start" do
     test "multiple proccesses succeed together", %{tenant: tenant} do
       parent = self()

test/realtime/tenants/replication_connection_test.exs

@@ -27,6 +27,23 @@ defmodule Realtime.Tenants.ReplicationConnectionTest do
     %{tenant: tenant}
   end
 
+  describe "temporary process" do
+    test "starts a temporary process", %{tenant: tenant} do
+      assert {:ok, pid} = ReplicationConnection.start(tenant, self())
+      assert conn = ReplicationConnection.whereis(tenant.external_id)
+
+      # Brutally kill the process
+      Process.exit(pid, :kill)
+      assert_process_down(pid)
+      assert_process_down(conn)
+      # Wait to ensure that the process has not restarted
+      Process.sleep(1000)
+
+      # Temporary process should not be registered
+      refute ReplicationConnection.whereis(tenant.external_id)
+    end
+  end
+
   describe "replication" do
     test "fails if tenant connection is invalid" do
       tenant =
@@ -387,4 +404,9 @@ defmodule Realtime.Tenants.ReplicationConnectionTest do
 
     Endpoint.subscribe(tenant_topic, metadata: fastlane)
   end
+
+  defp assert_process_down(pid, timeout \\ 100) do
+    ref = Process.monitor(pid)
+    assert_receive {:DOWN, ^ref, :process, ^pid, _reason}, timeout
+  end
 end