feat: workspace folders (#18)

This adds support for multiple projects within the server node, and
starts a project node for each detected project within the workspace.
The detection implemented here is just using the presence of a `mix.exs`
file to assume it's a project root.

It also changes the manager node to use the name of the workspace folder
instead of the name of the first project that starts distribution. The
reason is that if you have projects `a` and `b` in the same root folder
`foo`, and you start Expert on the root folder, then the manager node
will be named after the first project node that is started. Running
`epmd -names` shows this:

- `project-a-entropy`
- `project-b-entropy`
- `manager-a-entropy`

I htink having the manager be named `manager-foo-entropy` sounds more
correct.

To test this branch you can clone this repo:
https://github.com/doorgan/monorepo_test
Then check on files from both `main` and `secondary` that:
- Completions work
- Go to definition and workspace symbols work
- Code actions (like Refactorex refactors) work
- Instructing the LSP to reindex everything runs the indexer on all
projects

Additionally you can also test that Lexical works on `.exs` or `.ex`
files in a workspace without a `mix.exs` file.

It is worth noting that this should be tested with VS Code using the
Lexical LSP extension and pointing it to an expert release using this
branch, or some other editor (like emacs) that doesn't automatically
start a new Expert server(ie something other than Neovim)

Closes #136

Author: doorgan

apps/engine/benchmarks/ast_analyze.exs

@@ -1,3 +1,5 @@
+Mix.install([:benchee])
+
 alias Forge.Ast
 alias Forge.Document
 

apps/engine/benchmarks/enum_index.exs

@@ -1,3 +1,5 @@
+Mix.install([:benchee])
+
 alias Engine.Search.Indexer
 
 path =

apps/engine/benchmarks/ets_bench.exs

@@ -1,3 +1,5 @@
+Mix.install([:benchee])
+
 alias Forge.Project
 
 alias Engine.Search.Store.Backends.Ets

apps/engine/benchmarks/versions_bench.exs

@@ -1,3 +1,5 @@
+Mix.install([:benchee])
+
 alias Forge.VM.Versions
 
 Benchee.run(%{

apps/engine/mix.exs

@@ -44,7 +44,6 @@ defmodule Engine.MixProject do
 
   defp deps do
     [
-      {:benchee, "~> 1.3", only: :test},
       {:deps_nix, "~> 2.4", only: :dev},
       Mix.Credo.dependency(),
       Mix.Dialyzer.dependency(),

apps/expert/config/runtime.exs

@@ -20,5 +20,3 @@ if Code.ensure_loaded?(LoggerFileBackend) do
 else
   :ok
 end
-
-require Logger

apps/expert/lib/expert.ex

@@ -1,9 +1,11 @@
 defmodule Expert do
+  alias Expert.ActiveProjects
   alias Expert.Project
   alias Expert.Protocol.Convert
   alias Expert.Protocol.Id
   alias Expert.Provider.Handlers
   alias Expert.State
+  alias Forge.Project
   alias GenLSP.Enumerations
   alias GenLSP.Requests
   alias GenLSP.Structures
@@ -16,6 +18,7 @@ defmodule Expert do
     GenLSP.Notifications.TextDocumentDidChange,
     GenLSP.Notifications.WorkspaceDidChangeConfiguration,
     GenLSP.Notifications.WorkspaceDidChangeWatchedFiles,
+    GenLSP.Notifications.WorkspaceDidChangeWorkspaceFolders,
     GenLSP.Notifications.TextDocumentDidClose,
     GenLSP.Notifications.TextDocumentDidOpen,
     GenLSP.Notifications.TextDocumentDidSave,
@@ -54,17 +57,28 @@ defmodule Expert do
 
     with {:ok, response, state} <- State.initialize(state, request),
          {:ok, response} <- Expert.Protocol.Convert.to_lsp(response) do
-      Task.Supervisor.start_child(:expert_task_queue, fn ->
-        # dirty sleep to allow initialize response to return before progress reports
-        Process.sleep(50)
-        config = state.configuration
+      workspace_folders = request.params.workspace_folders || []
 
-        log_info(lsp, "Starting project")
+      projects =
+        for %{uri: uri} <- workspace_folders,
+            project = Project.new(uri),
+            # Only include Mix projects, or include single-folder workspaces with
+            # bare elixir files.
+            project.mix_project? || Project.elixir_project?(project) do
+          project
+        end
 
-        start_result = Project.Supervisor.start(config.project)
+      ActiveProjects.set_projects(projects)
 
-        send(Expert, {:engine_initialized, start_result})
-      end)
+      for project <- projects do
+        Task.Supervisor.start_child(:expert_task_queue, fn ->
+          log_info(lsp, project, "Starting project")
+
+          start_result = Expert.Project.Supervisor.ensure_node_started(project)
+
+          send(Expert, {:engine_initialized, project, start_result})
+        end)
+      end
 
       {:reply, response, assign(lsp, state: state)}
     else
@@ -109,43 +123,74 @@ defmodule Expert do
   def handle_request(request, lsp) do
     state = assigns(lsp).state
 
-    if state.engine_initialized? do
-      with {:ok, handler} <- fetch_handler(request),
-           {:ok, request} <- Convert.to_native(request),
-           {:ok, response} <- handler.handle(request, state.configuration),
-           {:ok, response} <- Expert.Protocol.Convert.to_lsp(response) do
-        {:reply, response, lsp}
-      else
-        {:error, {:unhandled, _}} ->
-          Logger.info("Unhandled request: #{request.method}")
-
-          {:reply,
-           %GenLSP.ErrorResponse{
-             code: GenLSP.Enumerations.ErrorCodes.method_not_found(),
-             message: "Method not found"
-           }, lsp}
-
-        error ->
-          message = "Failed to handle #{request.method}, #{inspect(error)}"
-          Logger.error(message)
-
-          {:reply,
-           %GenLSP.ErrorResponse{
-             code: GenLSP.Enumerations.ErrorCodes.internal_error(),
-             message: message
-           }, lsp}
-      end
+    with {:ok, handler} <- fetch_handler(request),
+         {:ok, request} <- Convert.to_native(request),
+         :ok <- check_engine_initialized(request),
+         {:ok, response} <- handler.handle(request, state.configuration),
+         {:ok, response} <- Expert.Protocol.Convert.to_lsp(response) do
+      {:reply, response, lsp}
     else
-      GenLSP.warning(
-        lsp,
-        "Received request #{request.method} before engine was initialized. Ignoring."
-      )
+      {:error, {:unhandled, _}} ->
+        Logger.info("Unhandled request: #{request.method}")
 
-      {:noreply, lsp}
+        {:reply,
+         %GenLSP.ErrorResponse{
+           code: GenLSP.Enumerations.ErrorCodes.method_not_found(),
+           message: "Method not found"
+         }, lsp}
+
+      {:error, :engine_not_initialized, project} ->
+        GenLSP.info(
+          lsp,
+          "Received request #{request.method} before engine for #{project && Project.name(project)} was initialized. Ignoring."
+        )
+
+        {:reply, nil, lsp}
+
+      error ->
+        message = "Failed to handle #{request.method}, #{inspect(error)}"
+        Logger.error(message)
+
+        {:reply,
+         %GenLSP.ErrorResponse{
+           code: GenLSP.Enumerations.ErrorCodes.internal_error(),
+           message: message
+         }, lsp}
     end
   end
 
+  defp check_engine_initialized(request) do
+    if document_request?(request) do
+      case Forge.Document.Container.context_document(request, nil) do
+        %Forge.Document{} = document ->
+          projects = ActiveProjects.projects()
+          project = Project.project_for_document(projects, document)
+
+          if ActiveProjects.active?(project) do
+            :ok
+          else
+            {:error, :engine_not_initialized, project}
+          end
+
+        nil ->
+          {:error, :engine_not_initialized, nil}
+      end
+    else
+      :ok
+    end
+  end
+
+  defp document_request?(%{document: %Forge.Document{}}), do: true
+
+  defp document_request?(%{params: params}) do
+    document_request?(params)
+  end
+
+  defp document_request?(%{text_document: %{uri: _}}), do: true
+  defp document_request?(_), do: false
+
   def handle_notification(%GenLSP.Notifications.Initialized{}, lsp) do
+    Logger.info("Server initialized, registering capabilities")
     registrations = registrations()
 
     if nil != GenLSP.request(lsp, registrations) do
@@ -189,35 +234,33 @@ defmodule Expert do
     end
   end
 
-  def handle_info({:engine_initialized, {:ok, _pid}}, lsp) do
-    state = assigns(lsp).state
-
-    new_state = %{state | engine_initialized?: true}
-
-    lsp = assign(lsp, state: new_state)
-
-    Logger.info("Engine initialized")
+  def handle_info({:engine_initialized, project, {:ok, _pid}}, lsp) do
+    log_info(
+      lsp,
+      project,
+      "Engine initialized for project #{Project.name(project)}"
+    )
 
     {:noreply, lsp}
   end
 
-  def handle_info({:engine_initialized, {:error, reason}}, lsp) do
+  def handle_info({:engine_initialized, project, {:error, reason}}, lsp) do
     error_message = initialization_error_message(reason)
-    log_error(lsp, error_message)
+    log_error(lsp, project, error_message)
 
     {:noreply, lsp}
   end
 
-  def log_info(lsp \\ get_lsp(), message) do
-    message = log_prepend_project_root(message, assigns(lsp).state)
+  def log_info(lsp \\ get_lsp(), project, message) do
+    message = log_prepend_project_root(message, project)
 
     Logger.info(message)
     GenLSP.info(lsp, message)
   end
 
   # When logging errors we also notify the client to display the message
-  def log_error(lsp \\ get_lsp(), message) do
-    message = log_prepend_project_root(message, assigns(lsp).state)
+  def log_error(lsp \\ get_lsp(), project, message) do
+    message = log_prepend_project_root(message, project)
 
     Logger.error(message)
     GenLSP.error(lsp, message)
@@ -335,11 +378,7 @@ defmodule Expert do
     end
   end
 
-  defp log_prepend_project_root(message, %State{
-         configuration: %Expert.Configuration{project: %Forge.Project{} = project}
-       }) do
+  defp log_prepend_project_root(message, project) do
     "[Project #{project.root_uri}] #{message}"
   end
-
-  defp log_prepend_project_root(message, _state), do: message
 end

apps/expert/lib/expert/active_projects.ex

@@ -0,0 +1,78 @@
+defmodule Expert.ActiveProjects do
+  @moduledoc """
+  A cache to keep track of active projects.
+
+  Since GenLSP events happen asynchronously, we use an ets table to keep track of
+  them and avoid race conditions when we try to update the list of active projects.
+  """
+  alias Forge.Project
+
+  use GenServer
+
+  def child_spec(_) do
+    %{
+      id: __MODULE__,
+      start: {__MODULE__, :start_link, []}
+    }
+  end
+
+  def start_link do
+    GenServer.start_link(__MODULE__, [], name: __MODULE__)
+  end
+
+  def init(_) do
+    __MODULE__ = :ets.new(__MODULE__, [:set, :named_table, :public, read_concurrency: true])
+
+    __MODULE__.Ready =
+      :ets.new(__MODULE__.Ready, [:set, :named_table, :public, read_concurrency: true])
+
+    {:ok, nil}
+  end
+
+  def projects do
+    __MODULE__
+    |> :ets.tab2list()
+    |> Enum.map(fn {_, project} -> project end)
+  end
+
+  @spec find_by_root_uri(Forge.uri()) :: Project.t() | nil
+  def find_by_root_uri(root_uri) do
+    case :ets.lookup(__MODULE__, root_uri) do
+      [{_, project}] -> project
+      [] -> nil
+    end
+  end
+
+  def add_projects(new_projects) when is_list(new_projects) do
+    for new_project <- new_projects do
+      # We use `:ets.insert_new/2` to avoid overwriting the cached project's entropy
+      :ets.insert_new(__MODULE__, {new_project.root_uri, new_project})
+    end
+  end
+
+  def remove_projects(removed_projects) when is_list(removed_projects) do
+    for removed_project <- removed_projects do
+      :ets.delete(__MODULE__, removed_project.root_uri)
+    end
+  end
+
+  def set_projects(new_projects) when is_list(new_projects) do
+    :ets.delete_all_objects(__MODULE__)
+    add_projects(new_projects)
+  end
+
+  def set_ready(%Project{} = project, ready?) when is_boolean(ready?) do
+    if ready? do
+      :ets.insert(__MODULE__.Ready, {project.root_uri, true})
+    else
+      :ets.delete(__MODULE__.Ready, project.root_uri)
+    end
+  end
+
+  def active?(%Project{} = project) do
+    case :ets.lookup(__MODULE__.Ready, project.root_uri) do
+      [{_, true}] -> true
+      _ -> false
+    end
+  end
+end

apps/expert/lib/expert/application.ex

@@ -117,6 +117,7 @@ defmodule Expert.Application do
       {GenLSP.Assigns, [name: Expert.Assigns]},
       {Task.Supervisor, name: :expert_task_queue},
       {GenLSP.Buffer, [name: Expert.Buffer] ++ buffer_opts},
+      {Expert.ActiveProjects, []},
       {Expert,
        name: Expert,
        buffer: Expert.Buffer,

apps/expert/lib/expert/configuration.ex

@@ -6,19 +6,16 @@ defmodule Expert.Configuration do
   alias Expert.Configuration.Support
   alias Expert.Dialyzer
   alias Expert.Protocol.Id
-  alias Forge.Project
   alias GenLSP.Notifications.WorkspaceDidChangeConfiguration
   alias GenLSP.Requests
   alias GenLSP.Structures
 
-  defstruct project: nil,
-            support: nil,
+  defstruct support: nil,
             client_name: nil,
             additional_watched_extensions: nil,
             dialyzer_enabled?: false
 
   @type t :: %__MODULE__{
-          project: Project.t() | nil,
           support: support | nil,
           client_name: String.t() | nil,
           additional_watched_extensions: [String.t()] | nil,
@@ -29,12 +26,11 @@ defmodule Expert.Configuration do
 
   @dialyzer {:nowarn_function, set_dialyzer_enabled: 2}
 
-  @spec new(Forge.uri(), map(), String.t() | nil) :: t
-  def new(root_uri, %Structures.ClientCapabilities{} = client_capabilities, client_name) do
+  @spec new(Structures.ClientCapabilities.t(), String.t() | nil) :: t
+  def new(%Structures.ClientCapabilities{} = client_capabilities, client_name) do
     support = Support.new(client_capabilities)
-    project = Project.new(root_uri)
 
-    %__MODULE__{support: support, project: project, client_name: client_name}
+    %__MODULE__{support: support, client_name: client_name}
     |> tap(&set/1)
   end
 
@@ -44,7 +40,6 @@ defmodule Expert.Configuration do
   end
 
   defp set(%__MODULE__{} = config) do
-    # FIXME(mhanberg): I don't think this will work once we have workspace support
     :persistent_term.put(__MODULE__, config)
   end