perf: optimise loading saved searches

lib/logflare/context_cache/cache_buster.ex

@@ -15,6 +15,7 @@ defmodule Logflare.ContextCache.CacheBuster do
   alias Logflare.Endpoints
   alias Logflare.PubSub
   alias Logflare.Rules
+  alias Logflare.SavedSearches
   alias Logflare.SourceSchemas
   alias Logflare.Sources
   alias Logflare.TeamUsers
@@ -156,6 +157,13 @@ defmodule Logflare.ContextCache.CacheBuster do
     {Endpoints, String.to_integer(id)}
   end
 
+  defp handle_record(%UpdatedRecord{
+         relation: {_schema, "saved_searches"},
+         record: %{"source_id" => source_id}
+       }) do
+    {SavedSearches, [source_id: String.to_integer(source_id)]}
+  end
+
   defp handle_record(%NewRecord{
          relation: {_schema, "billing_accounts"},
          record: %{"id" => _id}
@@ -229,6 +237,13 @@ defmodule Logflare.ContextCache.CacheBuster do
     {Auth, :not_found}
   end
 
+  defp handle_record(%NewRecord{
+         relation: {_schema, "saved_searches"},
+         record: %{"source_id" => source_id}
+       }) do
+    {SavedSearches, [source_id: String.to_integer(source_id)]}
+  end
+
   defp handle_record(%DeletedRecord{
          relation: {_schema, "billing_accounts"},
          old_record: %{"id" => id}
@@ -303,6 +318,13 @@ defmodule Logflare.ContextCache.CacheBuster do
     {Auth, String.to_integer(id)}
   end
 
+  defp handle_record(%DeletedRecord{
+         relation: {_schema, "saved_searches"},
+         old_record: %{"source_id" => source_id}
+       }) do
+    {SavedSearches, [source_id: String.to_integer(source_id)]}
+  end
+
   defp handle_record(_record) do
     :noop
   end

lib/logflare/context_cache/supervisor.ex

@@ -9,6 +9,7 @@ defmodule Logflare.ContextCache.Supervisor do
   alias Logflare.Billing
   alias Logflare.ContextCache
   alias Logflare.Backends
+  alias Logflare.SavedSearches
   alias Logflare.Sources
   alias Logflare.SourceSchemas
   alias Logflare.Users
@@ -66,7 +67,8 @@ defmodule Logflare.ContextCache.Supervisor do
       SourceSchemas.Cache,
       Auth.Cache,
       Endpoints.Cache,
-      Rules.Cache
+      Rules.Cache,
+      SavedSearches.Cache
     ]
   end
 

lib/logflare/saved_searches/cache.ex

@@ -0,0 +1,53 @@
+defmodule Logflare.SavedSearches.Cache do
+  @moduledoc false
+
+  alias Logflare.SavedSearches
+  alias Logflare.Utils
+
+  def child_spec(_) do
+    stats = Application.get_env(:logflare, :cache_stats, false)
+
+    %{
+      id: __MODULE__,
+      start:
+        {Cachex, :start_link,
+         [
+           __MODULE__,
+           [
+             hooks:
+               [
+                 if(stats, do: Utils.cache_stats()),
+                 Utils.cache_limit(100_000)
+               ]
+               |> Enum.filter(& &1),
+             expiration: Utils.cache_expiration_min()
+           ]
+         ]}
+    }
+  end
+
+  def list_saved_searches_by_user(user_id), do: apply_repo_fun(__ENV__.function, [user_id])
+
+  def bust_by(kw) do
+    kw
+    |> Enum.map(fn
+      {:source_id, source_id} ->
+        case Logflare.Sources.get(source_id) do
+          nil -> nil
+          source -> {:list_saved_searches_by_user, [source.user_id]}
+        end
+    end)
+    |> Enum.reject(&is_nil/1)
+    |> Enum.reduce(0, fn key, acc ->
+      case Cachex.take(__MODULE__, key) do
+        {:ok, nil} -> acc
+        {:ok, _value} -> acc + 1
+      end
+    end)
+    |> then(&{:ok, &1})
+  end
+
+  defp apply_repo_fun(arg1, arg2) do
+    Logflare.ContextCache.apply_fun(SavedSearches, arg1, arg2)
+  end
+end

lib/logflare/saved_searches/saved_searches.ex

@@ -50,15 +50,6 @@ defmodule Logflare.SavedSearches do
     |> Repo.insert()
   end
 
-  @doc """
-  Completely deletes a saved search.
-  TODO: remove, unused.
-  """
-  @spec delete(SavedSearch.t()) :: {:ok, SavedSearch.t()}
-  def delete(search) do
-    Repo.delete(search)
-  end
-
   @doc """
   Marks a SavedSearch as not saved (i.e. user will not see in dashboard)
   """
@@ -119,4 +110,15 @@ defmodule Logflare.SavedSearches do
     |> limit([s], 10)
     |> Repo.all()
   end
+
+  @spec list_saved_searches_by_user(number()) :: [SavedSearch.t()]
+  def list_saved_searches_by_user(user_id) do
+    SavedSearch
+    |> where([s], s.saved_by_user == true)
+    |> join(:inner, [s], src in Source, on: s.source_id == src.id)
+    |> where([s, src], src.user_id == ^user_id)
+    |> order_by([s, src], asc: src.name, asc: s.inserted_at)
+    |> preload([s, src], source: src)
+    |> Repo.all()
+  end
 end

lib/logflare/sources.ex

@@ -515,7 +515,6 @@ defmodule Logflare.Sources do
   def preload_for_dashboard(sources) do
     sources
     |> Enum.map(&preload_defaults/1)
-    |> Enum.map(&preload_saved_searches/1)
     |> Enum.map(&put_schema_field_count/1)
     |> Enum.sort_by(&{!&1.favorite, &1.name})
   end

lib/logflare_web/live/dashboard_live.ex

@@ -12,20 +12,22 @@ defmodule LogflareWeb.DashboardLive do
 
   @impl true
   def mount(_, _session, socket) do
+    %{user: user} = socket.assigns
+
     socket =
       socket
-      |> assign_new(:sources, fn %{user: user} ->
-        user
-        |> Sources.list_sources_by_user()
-        |> Sources.preload_for_dashboard()
-      end)
+      |> assign(
+        :sources,
+        user |> Sources.list_sources_by_user() |> Sources.preload_for_dashboard()
+      )
       |> assign_new(:source_metrics, fn %{sources: sources} ->
         sources
         |> Enum.into(%{}, fn source ->
           {to_string(source.token), %{metrics: source.metrics, updated_at: source.updated_at}}
         end)
       end)
-      |> assign_new(:plan, fn %{user: user} -> Billing.get_plan_by_user(user) end)
+      |> assign(:saved_searches, SavedSearches.Cache.list_saved_searches_by_user(user.id))
+      |> assign(:plan, Billing.get_plan_by_user(user))
       |> assign(:fade_in, false)
 
     if connected?(socket) do
@@ -55,19 +57,17 @@ defmodule LogflareWeb.DashboardLive do
   end
 
   def handle_event("delete_saved_search", %{"id" => search_id}, socket) do
-    %{user: user, sources: sources} = socket.assigns
+    %{user: user} = socket.assigns
 
     socket =
       with %Logflare.SavedSearch{source: source} = search <-
              SavedSearches.get(search_id) |> Repo.preload(:source),
            true <- Sources.get_by_user_access(user, source.id) |> is_struct(),
            {:ok, _response} <- SavedSearches.delete_by_user(search) do
-        sources =
-          sources
-          |> Sources.preload_saved_searches(force: true)
+        saved_searches = SavedSearches.list_saved_searches_by_user(user.id)
 
         socket
-        |> assign(sources: sources)
+        |> assign(saved_searches: saved_searches)
         |> put_flash(:info, "Saved search deleted!")
       else
         nil ->
@@ -127,7 +127,7 @@ defmodule LogflareWeb.DashboardLive do
       <div class="tw-max-w-[95%] tw-mx-auto">
         <div class="lg:tw-grid tw-grid-cols-12 tw-gap-8 tw-px-[15px] tw-mt-[50px]">
           <div class="tw-col-span-3">
-            <DashboardComponents.saved_searches sources={@sources} team={@team} />
+            <DashboardComponents.saved_searches saved_searches={@saved_searches} team={@team} />
             <DashboardComponents.members user={@user} team={@team} team_user={@team_user} />
           </div>
           <div class="tw-col-span-7">

lib/logflare_web/live/dashboard_live/dashboard_components.ex

@@ -139,31 +139,20 @@ defmodule LogflareWeb.DashboardLive.DashboardComponents do
   defp current_team_user?(_member, nil), do: false
   defp current_team_user?(member, team_user), do: member.provider_uid == team_user.provider_uid
 
-  attr :sources, :list, required: true
+  attr :saved_searches, :list, required: true
   attr :team, Logflare.Teams.Team, required: true
 
   def saved_searches(assigns) do
-    assigns =
-      assigns
-      |> assign(
-        :searches,
-        for(
-          source <- assigns.sources,
-          saved_search <- source.saved_searches,
-          do: {source, saved_search}
-        )
-      )
-
     ~H"""
     <div>
       <h5 class="header-margin">Saved Searches</h5>
-      <div :if={Enum.empty?(@searches)}>
+      <div :if={Enum.empty?(@saved_searches)}>
         Your saved searches will show up here. Save some searches!
       </div>
       <ul class="list-unstyled">
-        <li :for={{source, saved_search} <- @searches}>
-          <.team_link team={@team} href={~p"/sources/#{source}/search?#{%{querystring: saved_search.querystring, tailing: saved_search.tailing}}"} class="tw-text-white">
-            {source.name}:{saved_search.querystring}
+        <li :for={saved_search <- @saved_searches}>
+          <.team_link team={@team} href={~p"/sources/#{saved_search.source}/search?#{%{querystring: saved_search.querystring, tailing: saved_search.tailing}}"} class="tw-text-white">
+            {saved_search.source.name}:{saved_search.querystring}
           </.team_link>
           <span phx-click="delete_saved_search" phx-value-id={saved_search.id} data-confirm="Delete saved search?" class="tw-text-xs tw-ml-1.5 tw-text-white tw-cursor-pointer">
             <i class="fa fa-trash"></i>

priv/repo/migrations/20251216204334_add_saved_searches_source_id_inserted_at_index.exs

@@ -0,0 +1,10 @@
+defmodule Logflare.Repo.Migrations.AddSavedSearchesSourceIdInsertedAtIndex do
+  use Ecto.Migration
+
+  def change do
+    create index(:saved_searches, [:source_id, :inserted_at],
+             where: "saved_by_user = true",
+             name: :saved_searches_source_id_inserted_at_idx
+           )
+  end
+end

priv/repo/migrations/20251216234322_recreate_publications_add_saved_searches.exs

@@ -0,0 +1,20 @@
+defmodule Logflare.Repo.Migrations.RecreatePublicationsAddSavedSearches do
+  use Ecto.Migration
+
+  @publications Application.compile_env(:logflare, Logflare.ContextCache.CacheBuster)[
+                  :publications
+                ]
+  @table "saved_searches"
+
+  def up do
+    for p <- @publications do
+      execute("ALTER PUBLICATION #{p} ADD TABLE #{@table};")
+    end
+  end
+
+  def down do
+    for p <- @publications do
+      execute("ALTER PUBLICATION #{p} DROP TABLE #{@table};")
+    end
+  end
+end

test/logflare/saved_searches_test.exs

@@ -1,19 +1,19 @@
 defmodule Logflare.SavedSearchesTest do
   use Logflare.DataCase
-  alias Logflare.{SavedSearches, SavedSearch}
+  alias Logflare.{SavedSearches, SavedSearch, Repo}
 
   setup do
     user = insert(:user)
     source = insert(:source, user_id: user.id)
-    [source: source]
+    [user: user, source: source]
   end
 
   @valid_attrs %{lql_rules: [], querystring: "testing", saved_by_user: false, tailing: false}
   test "insert/2, get/1, delete/1, get_by_qs_source_id/1", %{source: source} do
     assert {:ok, %SavedSearch{} = saved_search} = SavedSearches.insert(@valid_attrs, source)
     assert saved_search == SavedSearches.get(saved_search.id)
     assert saved_search == SavedSearches.get_by_qs_source_id(saved_search.querystring, source.id)
-    assert {:ok, %SavedSearch{}} = SavedSearches.delete(saved_search)
+    assert {:ok, %SavedSearch{}} = Repo.delete(saved_search)
     assert nil == SavedSearches.get(saved_search.id)
   end
 
@@ -48,4 +48,44 @@ defmodule Logflare.SavedSearchesTest do
     assert [saved_search] == SavedSearches.suggest_saved_searches("test", source.id)
     assert [] == SavedSearches.suggest_saved_searches("other", source.id)
   end
+
+  test "list_saved_searches_by_user/1", %{user: user} do
+    other_user = insert(:user)
+
+    # Create sources with sortable names
+    source_b = insert(:source, user: user, name: "b-source")
+    source_a = insert(:source, user: user, name: "a-source")
+    source_c = insert(:source, user: user, name: "c-source")
+    other_source = insert(:source, user: other_user, name: "other-source")
+
+    {:ok, _search_b1} =
+      SavedSearches.insert(%{@valid_attrs | saved_by_user: true, querystring: "b1"}, source_b)
+
+    {:ok, _search_a1} =
+      SavedSearches.insert(%{@valid_attrs | saved_by_user: true, querystring: "a1"}, source_a)
+
+    {:ok, _search_a2} =
+      SavedSearches.insert(%{@valid_attrs | saved_by_user: true, querystring: "a2"}, source_a)
+
+    {:ok, _search_c1} =
+      SavedSearches.insert(%{@valid_attrs | saved_by_user: true, querystring: "c1"}, source_c)
+
+    # not saved by user: should not be returned
+    {:ok, _not_saved} =
+      SavedSearches.insert(
+        %{@valid_attrs | saved_by_user: false, querystring: "not-saved"},
+        source_a
+      )
+
+    # not owned by user: should not be returned
+    {:ok, _other_search} =
+      SavedSearches.insert(
+        %{@valid_attrs | saved_by_user: true, querystring: "other"},
+        other_source
+      )
+
+    result = SavedSearches.list_saved_searches_by_user(user.id)
+
+    assert ["a1", "a2", "b1", "c1"] == result |> Enum.map(& &1.querystring)
+  end
 end

test/logflare/sources_test.exs

@@ -221,7 +221,6 @@ defmodule Logflare.SourcesTest do
       sources = Sources.preload_for_dashboard(sources)
 
       assert Enum.all?(sources, &Ecto.assoc_loaded?(&1.user))
-      assert Enum.all?(sources, &Ecto.assoc_loaded?(&1.saved_searches))
 
       refute Enum.any?(sources, &Ecto.assoc_loaded?(&1.rules))
     end

test/logflare_web/live/dashboard_live_test.exs

@@ -101,8 +101,9 @@ defmodule LogflareWeb.DashboardLiveTest do
       |> element("[phx-click='delete_saved_search'][phx-value-id='#{saved_search.id}']")
       |> render_click()
 
-      {:ok, _view, html} = live(conn, "/dashboard")
+      Cachex.clear(Logflare.SavedSearches.Cache)
 
+      {:ok, _view, html} = live(conn, "/dashboard")
       refute html =~ saved_search.querystring
     end