feat: on the fly engine compilation

doorgan · doorgan · commit 714ac00b7dcb · 2025-08-22T00:08:38.000-03:00
diff --git a/apps/engine/.gitignore b/apps/engine/.gitignore
@@ -24,3 +24,6 @@ engine-*.tar
 
 # Temporary files, for example, from tests.
 /tmp/
+
+# Ignore the package zip built via `mix build --zip`.
+engine-*.zip
diff --git a/apps/engine/lib/engine/build/document/compilers/eex.ex b/apps/engine/lib/engine/build/document/compilers/eex.ex
@@ -95,7 +95,7 @@ defmodule Engine.Build.Document.Compilers.EEx do
   def do_eval_quoted(quoted_ast, path) do
     eval_heex_quoted? =
       quoted_ast
-      |> Future.Macro.path(&match?({:require, [context: Phoenix.LiveView.TagEngine], _}, &1))
+      |> Forge.Future.Macro.path(&match?({:require, [context: Phoenix.LiveView.TagEngine], _}, &1))
       |> then(&(not is_nil(&1)))
 
     env =
diff --git a/apps/engine/lib/engine/code_intelligence/definition.ex b/apps/engine/lib/engine/code_intelligence/definition.ex
@@ -8,9 +8,9 @@ defmodule Engine.CodeIntelligence.Definition do
   alias Forge.Document.Location
   alias Forge.Document.Position
   alias Forge.Formats
+  alias Forge.Future.Code
   alias Forge.Search.Indexer.Entry
   alias Forge.Text
-  alias Future.Code
 
   require Logger
 
diff --git a/apps/engine/lib/engine/code_intelligence/entity.ex b/apps/engine/lib/engine/code_intelligence/entity.ex
@@ -5,7 +5,7 @@ defmodule Engine.CodeIntelligence.Entity do
   alias Forge.Document.Position
   alias Forge.Document.Range
   alias Forge.Formats
-  alias Future.Code, as: Code
+  alias Forge.Future.Code, as: Code
 
   alias Sourceror.Zipper
 
diff --git a/apps/engine/lib/engine/module/loader.ex b/apps/engine/lib/engine/module/loader.ex
@@ -5,7 +5,7 @@ defmodule Engine.Module.Loader do
   is that having a cache will improve performance
   """
 
-  alias Future.Code
+  alias Forge.Future.Code
   use Agent
 
   def start_link(_) do
diff --git a/apps/engine/lib/engine/modules.ex b/apps/engine/lib/engine/modules.ex
@@ -4,7 +4,7 @@ defmodule Engine.Modules do
   """
 
   alias Engine.Module.Loader
-  alias Future.Code.Typespec
+  alias Forge.Future.Code.Typespec
 
   @typedoc "Module documentation record as defined by EEP-48"
   @type docs_v1 :: tuple()
@@ -146,7 +146,7 @@ defmodule Engine.Modules do
 
   defp format_definition(quoted) do
     quoted
-    |> Future.Code.quoted_to_algebra()
+    |> Forge.Future.Code.quoted_to_algebra()
     |> Inspect.Algebra.format(60)
     |> IO.iodata_to_binary()
   end
diff --git a/apps/expert/lib/expert.ex b/apps/expert/lib/expert.ex
@@ -23,6 +23,10 @@ defmodule Expert do
 
   @dialyzer {:nowarn_function, apply_to_state: 2}
 
+  @version Mix.Project.config()[:version]
+
+  def vsn, do: @version
+
   def get_lsp, do: :persistent_term.get(:expert_lsp, nil)
 
   def start_link(args) do
diff --git a/apps/expert/lib/expert/code_intelligence/completion.ex b/apps/expert/lib/expert/code_intelligence/completion.ex
@@ -8,8 +8,8 @@ defmodule Expert.CodeIntelligence.Completion do
   alias Forge.Ast.Env
   alias Forge.Completion.Candidate
   alias Forge.Document.Position
+  alias Forge.Future.Code, as: Code
   alias Forge.Project
-  alias Future.Code, as: Code
   alias GenLSP.Enumerations.CompletionTriggerKind
   alias GenLSP.Structures.CompletionContext
   alias GenLSP.Structures.CompletionItem
diff --git a/apps/expert/lib/expert/code_intelligence/completion/builder.ex b/apps/expert/lib/expert/code_intelligence/completion/builder.ex
@@ -16,7 +16,7 @@ defmodule Expert.CodeIntelligence.Completion.Builder do
   alias Forge.Document.Edit
   alias Forge.Document.Position
   alias Forge.Document.Range
-  alias Future.Code, as: Code
+  alias Forge.Future.Code, as: Code
   alias GenLSP.Enumerations.InsertTextFormat
   alias GenLSP.Enumerations.MarkupKind
   alias GenLSP.Structures.CompletionItem
diff --git a/apps/expert/lib/expert/code_intelligence/completion/translations/struct.ex b/apps/expert/lib/expert/code_intelligence/completion/translations/struct.ex
@@ -1,7 +1,7 @@
 defmodule Expert.CodeIntelligence.Completion.Translations.Struct do
   alias Forge.Ast.Env
   alias Forge.Formats
-  alias Future.Code, as: Code
+  alias Forge.Future.Code, as: Code
   alias GenLSP.Enumerations.CompletionItemKind
 
   def completion(%Env{} = _env, _builder, _module_name, _full_name, 0) do
diff --git a/apps/expert/lib/expert/code_intelligence/completion/translations/struct_field.ex b/apps/expert/lib/expert/code_intelligence/completion/translations/struct_field.ex
@@ -4,7 +4,7 @@ defmodule Expert.CodeIntelligence.Completion.Translations.StructField do
   alias Expert.CodeIntelligence.Completion.Translations
   alias Forge.Ast.Env
   alias Forge.Completion.Candidate
-  alias Future.Code, as: Code
+  alias Forge.Future.Code, as: Code
   alias GenLSP.Enumerations.CompletionItemKind
 
   defimpl Translatable, for: Candidate.StructField do
diff --git a/apps/expert/lib/expert/engine_node.ex b/apps/expert/lib/expert/engine_node.ex
@@ -115,7 +115,7 @@ defmodule Expert.EngineNode do
     bootstrap_args = [project, Document.Store.entropy(), all_app_configs()]
 
     with {:ok, node_pid} <- EngineSupervisor.start_project_node(project),
-         :ok <- start_node(project, glob_paths()),
+         :ok <- start_node(project, glob_paths(project)),
          :ok <- :rpc.call(node_name, Engine.Bootstrap, :init, bootstrap_args),
          :ok <- ensure_apps_started(node_name) do
       {:ok, node_name, node_pid}
@@ -152,7 +152,7 @@ defmodule Expert.EngineNode do
       ["/**/priv" | app_globs]
     end
 
-    def glob_paths do
+    def glob_paths(_) do
       for entry <- :code.get_path(),
           entry_string = List.to_string(entry),
           entry_string != ".",
@@ -165,9 +165,51 @@ defmodule Expert.EngineNode do
     # separately and copied to expert's priv directory.
     # When Engine is built in CI for a version matrix, we'll need to check if
     # we have the right version downloaded, and if not, we should download it.
-    defp glob_paths do
-      :expert
-      |> :code.priv_dir()
+    defp glob_paths(%Project{} = project) do
+      {:ok, elixir, _} = Expert.Port.elixir_executable(project)
+
+      engine_source =
+        System.get_env(
+          "EXPERT_ENGINE_PATH",
+          :expert |> :code.priv_dir() |> Path.join("engine_source/apps/engine")
+        )
+        |> Path.expand()
+
+      build_engine_script = :expert |> :code.priv_dir() |> Path.join("build_engine.exs")
+
+      opts =
+        [args: [elixir, build_engine_script, engine_source, Expert.vsn()], cd: engine_source]
+
+      launcher = Expert.Port.path()
+
+      port =
+        Port.open(
+          {:spawn_executable, launcher},
+          opts
+        )
+
+      wait_for_engine(port)
+    end
+
+    defp wait_for_engine(port) do
+      receive do
+        {^port, {:data, ~c"engine_path:" ++ engine_path}} ->
+          engine_path = engine_path |> to_string() |> String.trim()
+          Logger.info("Engine build available at: #{engine_path}")
+
+          ebin_paths(engine_path)
+
+        {^port, data} ->
+          wait_for_engine(port)
+
+        {:EXIT, ^port, reason} ->
+          Logger.error("Engine build script exited with reason: #{inspect(reason)}")
+          {:error, reason}
+      end
+    end
+
+    defp ebin_paths(base_path) do
+      base_path
       |> Path.join("lib/**/ebin")
       |> Path.wildcard()
     end
diff --git a/apps/expert/lib/expert/port.ex b/apps/expert/lib/expert/port.ex
@@ -148,8 +148,6 @@ defmodule Expert.Port do
   end
 
   def path({:unix, _}) do
-    require Logger
-
     with :non_existing <- :code.where_is_file(~c"port_wrapper.sh") do
       :expert
       |> :code.priv_dir()
diff --git a/apps/expert/lib/expert/release.ex b/apps/expert/lib/expert/release.ex
@@ -3,19 +3,61 @@ defmodule Expert.Release do
     Mix.Task.run(:namespace, [release.path])
 
     engine_path = Path.expand("../../../engine", __DIR__)
+    forge_path = Path.expand("../../../forge", __DIR__)
 
-    source = Path.join([engine_path, "_build/dev_ns"])
+    source = Path.join([engine_path, "_build/#{Mix.env()}_ns"])
+
+    engine_sources =
+      [
+        "lib",
+        "mix.exs",
+        "config",
+        "mix.lock"
+      ]
+      |> Enum.map(&Path.join([engine_path, &1]))
+
+    forge_sources =
+      [
+        "lib",
+        "mix.exs",
+        "config",
+        "mix.lock"
+      ]
+      |> Enum.map(&Path.join([forge_path, &1]))
+
+    root_exs = Path.expand("../../../../*.exs", __DIR__)
+    version_file = Path.expand("../../../../version.txt", __DIR__)
 
     dest =
       Path.join([
         release.path,
         "lib",
         "xp_expert-#{release.version}",
-        "priv"
+        "priv",
+        "engine_source"
       ])
 
     File.cp_r!(source, dest)
 
+    for source <- engine_sources do
+      dest_path = Path.join([dest, "apps", "engine", Path.basename(source)])
+      File.mkdir_p!(Path.dirname(dest_path))
+      File.cp_r!(source, dest_path)
+    end
+
+    for source <- forge_sources do
+      dest_path = Path.join([dest, "apps", "forge", Path.basename(source)])
+      File.mkdir_p!(Path.dirname(dest_path))
+      File.cp_r!(source, dest_path)
+    end
+
+    for exs_file <- Path.wildcard(root_exs) do
+      dest_path = Path.join([dest, Path.basename(exs_file)])
+      File.cp_r!(exs_file, dest_path)
+    end
+
+    File.cp!(version_file, Path.join([dest, "version.txt"]))
+
     release
   end
 end
diff --git a/apps/expert/priv/build_engine.exs b/apps/expert/priv/build_engine.exs
@@ -0,0 +1,39 @@
+{:ok, _} = Application.ensure_all_started(:elixir)
+{:ok, _} = Application.ensure_all_started(:mix)
+
+[engine_source_path, expert_vsn | _] = System.argv()
+
+major = :otp_release |> :erlang.system_info() |> List.to_string()
+version_file = Path.join([:code.root_dir(), "releases", major, "OTP_VERSION"])
+
+erlang_vsn =
+  try do
+    {:ok, contents} = File.read(version_file)
+    String.split(contents, "\n", trim: true)
+  else
+    [full] -> full
+    _ -> major
+  catch
+    :error ->
+      major
+  end
+
+elixir_vsn = System.version()
+
+out_path = :filename.basedir(:user_data, "Expert", %{version: expert_vsn})
+out_path = Path.join([out_path, "engine-#{elixir_vsn}-otp-#{erlang_vsn}"])
+dev_out_path = Path.join(out_path, "dev")
+
+if not File.exists?(dev_out_path) do
+  System.put_env("MIX_INSTALL_DIR", out_path)
+
+  Mix.Task.run("local.hex", ["--force"])
+  Mix.Task.run("local.rebar", ["--force"])
+  Mix.Project.in_project(:engine, engine_source_path, [build_path: out_path], fn _module ->
+    Mix.Task.run("deps.get", [])
+    Mix.Task.run("compile", [])
+    Mix.Task.run("namespace", [dev_out_path, out_path])
+  end)
+end
+
+IO.puts("engine_path:" <> dev_out_path)
diff --git a/apps/forge/lib/forge/ast.ex b/apps/forge/lib/forge/ast.ex
@@ -71,7 +71,7 @@ defmodule Forge.Ast do
   alias Forge.Document.Edit
   alias Forge.Document.Position
   alias Forge.Document.Range
-  alias Future.Code, as: Code
+  alias Forge.Future.Code, as: Code
   alias Sourceror.Zipper
 
   require Logger
@@ -292,7 +292,7 @@ defmodule Forge.Ast do
     case do_container_cursor_to_quoted(document_fragment) do
       {:ok, quoted} ->
         quoted
-        |> Future.Macro.path(&match?({:__cursor__, _, _}, &1))
+        |> Forge.Future.Macro.path(&match?({:__cursor__, _, _}, &1))
         |> List.wrap()
 
       _ ->
@@ -658,7 +658,7 @@ defmodule Forge.Ast do
     default
   end
 
-  # Similar to `Future.Macro.path/3`, but returns the path to the innermost
+  # Similar to `Forge.Future.Macro.path/3`, but returns the path to the innermost
   # node for which `fun` returns truthy instead of the path to the first node
   # that returns such.
   defp innermost_path(ast, acc, fun)
diff --git a/apps/forge/lib/forge/completion/candidate/argument_names.ex b/apps/forge/lib/forge/completion/candidate/argument_names.ex
@@ -8,7 +8,7 @@ defmodule Forge.Completion.Candidate.ArgumentNames do
   rendering the results useful for completion.
   """
 
-  alias Future.Code, as: Code
+  alias Forge.Future.Code, as: Code
 
   @type argument_names :: [String.t()]
   @default_specifier ~S(\\)
diff --git a/apps/forge/lib/forge/future/code.ex b/apps/forge/lib/forge/future/code.ex
@@ -1,7 +1,7 @@
 # Copied from https://github.com/elixir-lang/elixir/blob/bacea2cef6323d0ede4222f36ddcedd82cb514e4/lib/elixir/lib/code.ex
 # I commented print_diagnostic/1, because it is not used in the codebase.
 # And I changed string_to_quoted/2 to make it use :future_elixir
-defmodule Future.Code do
+defmodule Forge.Future.Code do
   @moduledoc ~S"""
   Utilities for managing code compilation, code evaluation, and code loading.
 
diff --git a/apps/forge/lib/forge/future/code/fragment.ex b/apps/forge/lib/forge/future/code/fragment.ex
@@ -1,6 +1,6 @@
 # Copied from https://raw.githubusercontent.com/elixir-lang/elixir/d7ea2fa2e4e5de1990297be19495fc93740b2e8b/lib/elixir/lib/code/fragment.ex
-defmodule Future.Code.Fragment do
-  alias Future.Code, as: Code
+defmodule Forge.Future.Code.Fragment do
+  alias Forge.Future.Code, as: Code
 
   @moduledoc """
   This module provides conveniences for analyzing fragments of
diff --git a/apps/forge/lib/forge/future/code/indentifier.ex b/apps/forge/lib/forge/future/code/indentifier.ex
@@ -1,5 +1,5 @@
 # Copied from https://github.com/elixir-lang/elixir/blob/d7ea2fa2e4e5de1990297be19495fc93740b2e8b/lib/elixir/lib/code/identifier.ex
-defmodule Future.Code.Identifier do
+defmodule Forge.Future.Code.Identifier do
   @moduledoc false
 
   @doc """
diff --git a/apps/forge/lib/forge/future/code/typespec.ex b/apps/forge/lib/forge/future/code/typespec.ex
@@ -1,5 +1,5 @@
 # Copied from https://github.com/elixir-lang/elixir/blob/d7ea2fa2e4e5de1990297be19495fc93740b2e8b/lib/elixir/lib/code/typespec.ex
-defmodule Future.Code.Typespec do
+defmodule Forge.Future.Code.Typespec do
   @moduledoc false
 
   @doc """
diff --git a/apps/forge/lib/forge/future/macro.ex b/apps/forge/lib/forge/future/macro.ex
@@ -2,7 +2,7 @@
 # only copied the `path/2`
 import Kernel, except: [to_string: 1]
 
-defmodule Future.Macro do
+defmodule Forge.Future.Macro do
   @doc """
   Returns the path to the node in `ast` which `fun` returns `true`.
 
diff --git a/apps/forge/lib/forge/future/mix/tasks/format.ex b/apps/forge/lib/forge/future/mix/tasks/format.ex
diff --git a/apps/forge/lib/mix/tasks/namespace.ex b/apps/forge/lib/mix/tasks/namespace.ex
@@ -30,7 +30,11 @@ defmodule Mix.Tasks.Namespace do
 
   require Logger
 
-  def run([base_directory]) do
+  def run([base_directory | opts]) do
+    cwd = List.first(opts)
+
+    :persistent_term.put(:forge_namespace_cwd, cwd || File.cwd!())
+
     # Ensure we cache the loaded apps at the time of namespacing
     # Otherwise only the @extra_apps will be cached
     init()
@@ -124,7 +128,7 @@ defmodule Mix.Tasks.Namespace do
   end
 
   defp discover_deps_apps do
-    cwd = File.cwd!()
+    cwd = :persistent_term.get(:forge_namespace_cwd, File.cwd!())
 
     :application.loaded_applications()
     |> Enum.flat_map(fn {app_name, _description, _version} ->
