feat: on the fly engine compilation

Author: doorgan

apps/engine/.gitignore

@@ -24,3 +24,6 @@ engine-*.tar
 
 # Temporary files, for example, from tests.
 /tmp/
+
+# Ignore the package zip built via `mix build --zip`.
+engine-*.zip

apps/engine/lib/engine/completion.ex

@@ -1,11 +1,11 @@
 defmodule Engine.Completion do
   alias Forge.Ast.Analysis
   alias Forge.Ast.Env
+  alias Forge.Completion.Candidate
   alias Forge.Document
   alias Forge.Document.Position
 
   alias Engine.CodeMod.Format
-  alias Forge.Completion.Candidate
 
   import Document.Line
   import Forge.Logging

apps/expert/lib/expert.ex

@@ -23,6 +23,10 @@ defmodule Expert do
 
   @dialyzer {:nowarn_function, apply_to_state: 2}
 
+  @version Mix.Project.config()[:version]
+
+  def vsn, do: @version
+
   def get_lsp, do: :persistent_term.get(:expert_lsp, nil)
 
   def start_link(args) do

apps/expert/lib/expert/engine_node.ex

@@ -115,7 +115,8 @@ defmodule Expert.EngineNode do
     bootstrap_args = [project, Document.Store.entropy(), all_app_configs()]
 
     with {:ok, node_pid} <- EngineSupervisor.start_project_node(project),
-         :ok <- start_node(project, glob_paths()),
+         {:ok, glob_paths} <- glob_paths(project),
+         :ok <- start_node(project, glob_paths),
          :ok <- :rpc.call(node_name, Engine.Bootstrap, :init, bootstrap_args),
          :ok <- ensure_apps_started(node_name) do
       {:ok, node_name, node_pid}
@@ -152,22 +153,79 @@ defmodule Expert.EngineNode do
       ["/**/priv" | app_globs]
     end
 
-    def glob_paths do
-      for entry <- :code.get_path(),
-          entry_string = List.to_string(entry),
-          entry_string != ".",
-          Enum.any?(app_globs(), &PathGlob.match?(entry_string, &1, match_dot: true)) do
-        entry
-      end
+    def glob_paths(_) do
+      entries =
+        for entry <- :code.get_path(),
+            entry_string = List.to_string(entry),
+            entry_string != ".",
+            Enum.any?(app_globs(), &PathGlob.match?(entry_string, &1, match_dot: true)) do
+          entry
+        end
+
+      {:ok, entries}
     end
   else
-    # In dev and prod environments, a default build of Engine is built
-    # separately and copied to expert's priv directory.
-    # When Engine is built in CI for a version matrix, we'll need to check if
-    # we have the right version downloaded, and if not, we should download it.
-    defp glob_paths do
-      :expert
-      |> :code.priv_dir()
+    # In dev and prod environments, the engine source code is included in the
+    # Expert release, and we build it on the fly for the project elixir+opt
+    # versions if it was not built yet.
+    defp glob_paths(%Project{} = project) do
+      {:ok, elixir, _} = Expert.Port.elixir_executable(project)
+
+      expert_priv = :code.priv_dir(:expert)
+      packaged_engine_source = Path.join([expert_priv, "engine_source", "apps", "engine"])
+
+      engine_source =
+        "EXPERT_ENGINE_PATH"
+        |> System.get_env(packaged_engine_source)
+        |> Path.expand()
+
+      build_engine_script = Path.join(expert_priv, "build_engine.exs")
+
+      opts =
+        [
+          args: [
+            elixir,
+            build_engine_script,
+            "--source-path",
+            engine_source,
+            "--vsn",
+            Expert.vsn()
+          ],
+          cd: engine_source
+        ]
+
+      launcher = Expert.Port.path()
+
+      Logger.info("Finding or building engine for project #{Project.name(project)}")
+
+      port =
+        Port.open(
+          {:spawn_executable, launcher},
+          opts
+        )
+
+      wait_for_engine(port)
+    end
+
+    defp wait_for_engine(port) do
+      receive do
+        {^port, {:data, ~c"engine_path:" ++ engine_path}} ->
+          engine_path = engine_path |> to_string() |> String.trim()
+          Logger.info("Engine build available at: #{engine_path}")
+
+          {:ok, ebin_paths(engine_path)}
+
+        {^port, _data} ->
+          wait_for_engine(port)
+
+        {:EXIT, ^port, reason} ->
+          Logger.error("Engine build script exited with reason: #{inspect(reason)}")
+          {:error, reason}
+      end
+    end
+
+    defp ebin_paths(base_path) do
+      base_path
       |> Path.join("lib/**/ebin")
       |> Path.wildcard()
     end

apps/expert/lib/expert/port.ex

@@ -148,8 +148,6 @@ defmodule Expert.Port do
   end
 
   def path({:unix, _}) do
-    require Logger
-
     with :non_existing <- :code.where_is_file(~c"port_wrapper.sh") do
       :expert
       |> :code.priv_dir()

apps/expert/lib/expert/release.ex

@@ -2,19 +2,59 @@ defmodule Expert.Release do
   def assemble(release) do
     Mix.Task.run(:namespace, [release.path])
 
-    engine_path = Path.expand("../../../engine", __DIR__)
+    expert_root = Path.expand("../../../..", __DIR__)
+    engine_path = Path.join([expert_root, "apps", "engine"])
+    forge_path = Path.join([expert_root, "apps", "forge"])
 
-    source = Path.join([engine_path, "_build/dev_ns"])
+    engine_sources =
+      [
+        "lib",
+        "deps",
+        "mix.exs",
+        "config",
+        "mix.lock"
+      ]
+      |> Enum.map(&Path.join([engine_path, &1]))
+
+    forge_sources =
+      [
+        "lib",
+        "mix.exs",
+        "config",
+        "mix.lock"
+      ]
+      |> Enum.map(&Path.join([forge_path, &1]))
+
+    root_exs = Path.join([expert_root, "*.exs"])
+    version_file = Path.join([expert_root, "version.txt"])
 
     dest =
       Path.join([
         release.path,
         "lib",
         "xp_expert-#{release.version}",
-        "priv"
+        "priv",
+        "engine_source"
       ])
 
-    File.cp_r!(source, dest)
+    for source <- engine_sources do
+      dest_path = Path.join([dest, "apps", "engine", Path.basename(source)])
+      File.mkdir_p!(Path.dirname(dest_path))
+      File.cp_r!(source, dest_path)
+    end
+
+    for source <- forge_sources do
+      dest_path = Path.join([dest, "apps", "forge", Path.basename(source)])
+      File.mkdir_p!(Path.dirname(dest_path))
+      File.cp_r!(source, dest_path)
+    end
+
+    for exs_file <- Path.wildcard(root_exs) do
+      dest_path = Path.join([dest, Path.basename(exs_file)])
+      File.cp_r!(exs_file, dest_path)
+    end
+
+    File.cp!(version_file, Path.join([dest, "version.txt"]))
 
     release
   end

apps/expert/priv/build_engine.exs

@@ -0,0 +1,48 @@
+{:ok, _} = Application.ensure_all_started(:elixir)
+{:ok, _} = Application.ensure_all_started(:mix)
+
+{args, _, _} =
+  OptionParser.parse(
+    System.argv(),
+    strict: [
+      vsn: :string,
+      source_path: :string
+    ]
+  )
+
+expert_vsn = Keyword.fetch!(args, :vsn)
+engine_source_path = Keyword.fetch!(args, :source_path)
+
+major = :otp_release |> :erlang.system_info() |> List.to_string()
+version_file = Path.join([:code.root_dir(), "releases", major, "OTP_VERSION"])
+
+erlang_vsn =
+  try do
+    {:ok, contents} = File.read(version_file)
+    String.split(contents, "\n", trim: true)
+  else
+    [full] -> full
+    _ -> major
+  catch
+    :error ->
+      major
+  end
+
+elixir_vsn = System.version()
+
+user_data_path = :filename.basedir(:user_data, "Expert", %{version: expert_vsn})
+out_path = Path.join([user_data_path, "engine-#{elixir_vsn}-otp-#{erlang_vsn}"])
+build_path = Path.join(out_path, "dev")
+
+if not File.exists?(build_path) do
+  System.put_env("MIX_INSTALL_DIR", out_path)
+
+  Mix.Task.run("local.hex", ["--force"])
+  Mix.Task.run("local.rebar", ["--force"])
+  Mix.Project.in_project(:engine, engine_source_path, [build_path: out_path], fn _module ->
+    Mix.Task.run("compile", [])
+    Mix.Task.run("namespace", [build_path, "--cwd", out_path])
+  end)
+end
+
+IO.puts("engine_path:" <> build_path)

apps/forge/lib/mix/tasks/namespace.ex

@@ -30,7 +30,16 @@ defmodule Mix.Tasks.Namespace do
 
   require Logger
 
-  def run([base_directory]) do
+  def run([base_directory | opts]) do
+    {args, _, _} =
+      OptionParser.parse(opts,
+        strict: [cwd: :string]
+      )
+
+    cwd = Keyword.get(args, :cwd, File.cwd!())
+
+    :persistent_term.put(:forge_namespace_cwd, cwd)
+
     # Ensure we cache the loaded apps at the time of namespacing
     # Otherwise only the @extra_apps will be cached
     init()
@@ -124,7 +133,7 @@ defmodule Mix.Tasks.Namespace do
   end
 
   defp discover_deps_apps do
-    cwd = File.cwd!()
+    cwd = :persistent_term.get(:forge_namespace_cwd, File.cwd!())
 
     :application.loaded_applications()
     |> Enum.flat_map(fn {app_name, _description, _version} ->

justfile

@@ -49,24 +49,9 @@ lint *project="all":
   just mix {{ project }} credo
   just mix {{ project }} dialyzer
 
-build-engine:
-    #!/usr/bin/env bash
-    set -euxo pipefail
-
-    cd apps/engine
-    MIX_ENV=dev mix compile
-    namespaced_dir=_build/dev_ns/
-    rm -rf $namespaced_dir
-    mkdir -p $namespaced_dir
-
-    cp -a _build/dev/. "$namespaced_dir"
-
-    MIX_ENV=dev mix namespace "$namespaced_dir"
-
-
 [doc('Build a release for the local system')]
 [unix]
-release-local: (deps "expert") (compile "engine") build-engine
+release-local: (deps "engine") (deps "expert")
   #!/usr/bin/env bash
   cd apps/expert
 
@@ -79,18 +64,18 @@ release-local: (deps "expert") (compile "engine") build-engine
   MIX_ENV={{ env('MIX_ENV', 'prod')}} EXPERT_RELEASE_MODE=burrito BURRITO_TARGET="{{ local_target }}" mix release --overwrite
 
 [windows]
-release-local: (deps "expert") (compile "engine") build-engine
+release-local: (deps "engine") (deps "expert")
     # idk actually how to set env vars like this on windows, might crash
-    EXPERT_RELEASE_MODE=burrito BURRITO_TARGET="windows_amd64" MIX_ENV={{ env('MIX_ENV', 'prod')}} mix release --no-compile
+    EXPERT_RELEASE_MODE=burrito BURRITO_TARGET="windows_amd64" MIX_ENV={{ env('MIX_ENV', 'prod')}} mix release --overwrite
 
 [doc('Build releases for all target platforms')]
-release-all: (deps "expert") (compile "engine") build-engine
+release-all: (deps "engine") (deps "expert")
     #!/usr/bin/env bash
     cd apps/expert
-    EXPERT_RELEASE_MODE=burrito MIX_ENV={{ env('MIX_ENV', 'prod')}} mix release --no-compile
+    EXPERT_RELEASE_MODE=burrito MIX_ENV={{ env('MIX_ENV', 'prod')}} mix release --overwrite
 
 [doc('Build a plain release without burrito')]
-release-plain: (compile "engine")
+release-plain: (deps "engine") (deps "expert")
     #!/usr/bin/env bash
     cd apps/expert
     MIX_ENV={{ env('MIX_ENV', 'prod')}} mix release plain --overwrite