feat: on the fly engine builds (#24)

* feat: on the fly engine compilation

* chore: use Mix.install instead of mix compile

Also don't mark engine dependencies as optional, otherwise Mix.install
won't compile them and the Engine will fail to start

Author: doorgan

apps/engine/lib/engine/completion.ex

@@ -1,11 +1,11 @@
 defmodule Engine.Completion do
   alias Forge.Ast.Analysis
   alias Forge.Ast.Env
+  alias Forge.Completion.Candidate
   alias Forge.Document
   alias Forge.Document.Position
 
   alias Engine.CodeMod.Format
-  alias Forge.Completion.Candidate
 
   import Document.Line
   import Forge.Logging

apps/engine/mix.exs

@@ -51,9 +51,9 @@ defmodule Engine.MixProject do
        github: "elixir-lsp/elixir_sense", ref: "e3ddc403554050221a2fd19a10a896fa7525bc02"},
       {:forge, path: "../forge"},
       {:gen_lsp, "~> 0.11"},
-      {:patch, "~> 0.15", only: [:dev, :test], optional: true, runtime: false},
-      {:path_glob, "~> 0.2", optional: true},
-      {:phoenix_live_view, "~> 1.0", only: [:test], optional: true, runtime: false},
+      {:patch, "~> 0.15", only: [:dev, :test], runtime: false},
+      {:path_glob, "~> 0.2"},
+      {:phoenix_live_view, "~> 1.0", only: [:test], runtime: false},
       {:sourceror, "~> 1.9"},
       {:stream_data, "~> 1.1", only: [:test], runtime: false},
       {:refactorex, "~> 0.1.52"}

apps/expert/lib/expert.ex

@@ -23,6 +23,10 @@ defmodule Expert do
 
   @dialyzer {:nowarn_function, apply_to_state: 2}
 
+  @version Mix.Project.config()[:version]
+
+  def vsn, do: @version
+
   def get_lsp, do: :persistent_term.get(:expert_lsp, nil)
 
   def start_link(args) do

apps/expert/lib/expert/engine_node.ex

@@ -115,7 +115,8 @@ defmodule Expert.EngineNode do
     bootstrap_args = [project, Document.Store.entropy(), all_app_configs()]
 
     with {:ok, node_pid} <- EngineSupervisor.start_project_node(project),
-         :ok <- start_node(project, glob_paths()),
+         {:ok, glob_paths} <- glob_paths(project),
+         :ok <- start_node(project, glob_paths),
          :ok <- :rpc.call(node_name, Engine.Bootstrap, :init, bootstrap_args),
          :ok <- ensure_apps_started(node_name) do
       {:ok, node_name, node_pid}
@@ -152,22 +153,80 @@ defmodule Expert.EngineNode do
       ["/**/priv" | app_globs]
     end
 
-    def glob_paths do
-      for entry <- :code.get_path(),
-          entry_string = List.to_string(entry),
-          entry_string != ".",
-          Enum.any?(app_globs(), &PathGlob.match?(entry_string, &1, match_dot: true)) do
-        entry
-      end
+    def glob_paths(_) do
+      entries =
+        for entry <- :code.get_path(),
+            entry_string = List.to_string(entry),
+            entry_string != ".",
+            Enum.any?(app_globs(), &PathGlob.match?(entry_string, &1, match_dot: true)) do
+          entry
+        end
+
+      {:ok, entries}
     end
   else
-    # In dev and prod environments, a default build of Engine is built
-    # separately and copied to expert's priv directory.
-    # When Engine is built in CI for a version matrix, we'll need to check if
-    # we have the right version downloaded, and if not, we should download it.
-    defp glob_paths do
-      :expert
-      |> :code.priv_dir()
+    # In dev and prod environments, the engine source code is included in the
+    # Expert release, and we build it on the fly for the project elixir+opt
+    # versions if it was not built yet.
+    defp glob_paths(%Project{} = project) do
+      {:ok, elixir, _} = Expert.Port.elixir_executable(project)
+
+      expert_priv = :code.priv_dir(:expert)
+      packaged_engine_source = Path.join([expert_priv, "engine_source", "apps", "engine"])
+
+      engine_source =
+        "EXPERT_ENGINE_PATH"
+        |> System.get_env(packaged_engine_source)
+        |> Path.expand()
+
+      build_engine_script = Path.join(expert_priv, "build_engine.exs")
+
+      opts =
+        [
+          :stderr_to_stdout,
+          args: [
+            elixir,
+            build_engine_script,
+            "--source-path",
+            engine_source,
+            "--vsn",
+            Expert.vsn()
+          ],
+          cd: engine_source
+        ]
+
+      launcher = Expert.Port.path()
+
+      Logger.info("Finding or building engine for project #{Project.name(project)}")
+
+      port =
+        Port.open(
+          {:spawn_executable, launcher},
+          opts
+        )
+
+      wait_for_engine(port)
+    end
+
+    defp wait_for_engine(port) do
+      receive do
+        {^port, {:data, ~c"engine_path:" ++ engine_path}} ->
+          engine_path = engine_path |> to_string() |> String.trim()
+          Logger.info("Engine build available at: #{engine_path}")
+
+          {:ok, ebin_paths(engine_path)}
+
+        {^port, _data} ->
+          wait_for_engine(port)
+
+        {:EXIT, ^port, reason} ->
+          Logger.error("Engine build script exited with reason: #{inspect(reason)}")
+          {:error, reason}
+      end
+    end
+
+    defp ebin_paths(base_path) do
+      base_path
       |> Path.join("lib/**/ebin")
       |> Path.wildcard()
     end

apps/expert/lib/expert/port.ex

@@ -148,8 +148,6 @@ defmodule Expert.Port do
   end
 
   def path({:unix, _}) do
-    require Logger
-
     with :non_existing <- :code.where_is_file(~c"port_wrapper.sh") do
       :expert
       |> :code.priv_dir()

apps/expert/lib/expert/release.ex

@@ -2,19 +2,59 @@ defmodule Expert.Release do
   def assemble(release) do
     Mix.Task.run(:namespace, [release.path])
 
-    engine_path = Path.expand("../../../engine", __DIR__)
+    expert_root = Path.expand("../../../..", __DIR__)
+    engine_path = Path.join([expert_root, "apps", "engine"])
+    forge_path = Path.join([expert_root, "apps", "forge"])
 
-    source = Path.join([engine_path, "_build/dev_ns"])
+    engine_sources =
+      [
+        "lib",
+        "deps",
+        "mix.exs",
+        "config",
+        "mix.lock"
+      ]
+      |> Enum.map(&Path.join([engine_path, &1]))
+
+    forge_sources =
+      [
+        "lib",
+        "mix.exs",
+        "config",
+        "mix.lock"
+      ]
+      |> Enum.map(&Path.join([forge_path, &1]))
+
+    root_exs = Path.join([expert_root, "*.exs"])
+    version_file = Path.join([expert_root, "version.txt"])
 
     dest =
       Path.join([
         release.path,
         "lib",
         "xp_expert-#{release.version}",
-        "priv"
+        "priv",
+        "engine_source"
       ])
 
-    File.cp_r!(source, dest)
+    for source <- engine_sources do
+      dest_path = Path.join([dest, "apps", "engine", Path.basename(source)])
+      File.mkdir_p!(Path.dirname(dest_path))
+      File.cp_r!(source, dest_path)
+    end
+
+    for source <- forge_sources do
+      dest_path = Path.join([dest, "apps", "forge", Path.basename(source)])
+      File.mkdir_p!(Path.dirname(dest_path))
+      File.cp_r!(source, dest_path)
+    end
+
+    for exs_file <- Path.wildcard(root_exs) do
+      dest_path = Path.join([dest, Path.basename(exs_file)])
+      File.cp_r!(exs_file, dest_path)
+    end
+
+    File.cp!(version_file, Path.join([dest, "version.txt"]))
 
     release
   end

apps/expert/priv/build_engine.exs

@@ -0,0 +1,39 @@
+{:ok, _} = Application.ensure_all_started(:elixir)
+{:ok, _} = Application.ensure_all_started(:mix)
+
+{args, _, _} =
+  OptionParser.parse(
+    System.argv(),
+    strict: [
+      vsn: :string,
+      source_path: :string
+    ]
+  )
+
+expert_vsn = Keyword.fetch!(args, :vsn)
+engine_source_path = Keyword.fetch!(args, :source_path)
+
+expert_data_path = :filename.basedir(:user_data, "Expert", %{version: expert_vsn})
+
+System.put_env("MIX_INSTALL_DIR", expert_data_path)
+
+Mix.Task.run("local.hex", ["--force"])
+Mix.Task.run("local.rebar", ["--force"])
+
+Mix.install([{:engine, path: engine_source_path, env: :dev}],
+  start_applications: false,
+  config_path: Path.join(engine_source_path, "config/config.exs"),
+  lockfile: Path.join(engine_source_path, "mix.lock")
+)
+
+install_path = Mix.install_project_dir()
+
+dev_build_path = Path.join([install_path, "_build", "dev"])
+ns_build_path = Path.join([install_path, "_build", "dev_ns"])
+
+File.rm_rf!(ns_build_path)
+File.cp_r!(dev_build_path, ns_build_path)
+
+Mix.Task.run("namespace", [ns_build_path, "--cwd", install_path])
+
+IO.puts("engine_path:" <> ns_build_path)

apps/forge/lib/mix/tasks/namespace.ex

@@ -30,7 +30,16 @@ defmodule Mix.Tasks.Namespace do
 
   require Logger
 
-  def run([base_directory]) do
+  def run([base_directory | opts]) do
+    {args, _, _} =
+      OptionParser.parse(opts,
+        strict: [cwd: :string]
+      )
+
+    cwd = Keyword.get(args, :cwd, File.cwd!())
+
+    :persistent_term.put(:forge_namespace_cwd, cwd)
+
     # Ensure we cache the loaded apps at the time of namespacing
     # Otherwise only the @extra_apps will be cached
     init()
@@ -124,7 +133,7 @@ defmodule Mix.Tasks.Namespace do
   end
 
   defp discover_deps_apps do
-    cwd = File.cwd!()
+    cwd = :persistent_term.get(:forge_namespace_cwd, File.cwd!())
 
     :application.loaded_applications()
     |> Enum.flat_map(fn {app_name, _description, _version} ->

justfile

@@ -49,24 +49,9 @@ lint *project="all":
   just mix {{ project }} credo
   just mix {{ project }} dialyzer
 
-build-engine:
-    #!/usr/bin/env bash
-    set -euxo pipefail
-
-    cd apps/engine
-    MIX_ENV=dev mix compile
-    namespaced_dir=_build/dev_ns/
-    rm -rf $namespaced_dir
-    mkdir -p $namespaced_dir
-
-    cp -a _build/dev/. "$namespaced_dir"
-
-    MIX_ENV=dev mix namespace "$namespaced_dir"
-
-
 [doc('Build a release for the local system')]
 [unix]
-release-local: (deps "expert") (compile "engine") build-engine
+release-local: (deps "engine") (deps "expert")
   #!/usr/bin/env bash
   cd apps/expert
 
@@ -79,12 +64,12 @@ release-local: (deps "expert") (compile "engine") build-engine
   MIX_ENV={{ env('MIX_ENV', 'prod')}} EXPERT_RELEASE_MODE=burrito BURRITO_TARGET="{{ local_target }}" mix release --overwrite
 
 [windows]
-release-local: (deps "expert") (compile "engine") build-engine
+release-local: (deps "engine") (deps "expert")
     # idk actually how to set env vars like this on windows, might crash
-    EXPERT_RELEASE_MODE=burrito BURRITO_TARGET="windows_amd64" MIX_ENV={{ env('MIX_ENV', 'prod')}} mix release --no-compile
+    EXPERT_RELEASE_MODE=burrito BURRITO_TARGET="windows_amd64" MIX_ENV={{ env('MIX_ENV', 'prod')}} mix release --overwrite
 
 [doc('Build releases for all target platforms')]
-release-all: (deps "expert") (compile "engine") build-engine
+release-all: (deps "engine") (deps "expert")
     #!/usr/bin/env bash
     cd apps/expert
 
@@ -93,7 +78,7 @@ release-all: (deps "expert") (compile "engine") build-engine
     EXPERT_RELEASE_MODE=burrito MIX_ENV={{ env('MIX_ENV', 'prod')}} mix release --overwrite
 
 [doc('Build a plain release without burrito')]
-release-plain: (compile "engine")
+release-plain: (deps "engine") (deps "expert")
     #!/usr/bin/env bash
     cd apps/expert
     MIX_ENV={{ env('MIX_ENV', 'prod')}} mix release plain --overwrite