feat: use closest available engine version for the project node

Author: doorgan

.github/workflows/release.yml

@@ -17,7 +17,7 @@ jobs:
       release_created: ${{ steps.release.outputs.release_created }}
       tag_name: ${{ steps.release.outputs.tag_name }}
     steps:
-        id: release
+      id: release
 
   draft:
     name: draft
@@ -59,9 +59,64 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.token }}
         run: gh release upload ${{ needs.release.outputs.tag_name }} ./apps/expert/burrito_out/*
 
+  build_engine:
+    needs: [release, draft]
+    runs-on: ubuntu-latest
+    name: build engine
+    strategy:
+      matrix:
+        include:
+          - elixir: "1.18.1"
+            otp: "27"
+          - elixir: "1.18.1"
+            otp: "26"
+          - elixir: "1.17"
+            otp: "27"
+          - elixir: "1.17"
+            otp: "26"
+          - elixir: "1.17"
+            otp: "25"
+          - elixir: "1.16"
+            otp: "26"
+          - elixir: "1.16"
+            otp: "25"
+          - elixir: "1.15.8"
+            otp: "26"
+          - elixir: "1.15.8"
+            otp: "25"
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Elixir
+        uses: erlef/setup-beam@v1
+        with:
+          otp-version: ${{ matrix.otp }}
+          elixir-version: ${{ matrix.elixir }}
+
+      - name: Cache deps
+        id: cache-deps
+        uses: actions/cache@v3
+        with:
+          path: |
+            apps/**/deps
+            apps/**/_build
+
+          key: ${{ runner.os }}-mix-${{ matrix.elixir }}-${{ matrix.otp }}-${{ hashFiles('apps/**/mix.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-mix-${{ matrix.elixir }}-${{ matrix.otp }}-
+
+      - name: Build Engine
+        run: make build.engine.zip
+
+      - name: Upload Engine
+        env:
+          GITHUB_TOKEN: ${{ secrets.token }}
+        run: gh release upload ${{ needs.release.outputs.tag_name }} ./apps/engine/engine-*.zip
+
   publish:
     name: publish
-    needs: [release, draft, build]
+    needs: [release, draft, build, build_engine]
     runs-on: ubuntu-latest
     env:
       GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

Makefile

@@ -73,6 +73,9 @@ dialyzer.poncho: compile.poncho compile.protocols.poncho
 build.engine:
 	cd apps/engine && mix deps.get && MIX_ENV=dev mix build
 
+build.engine.zip:
+	cd apps/engine && mix deps.get && MIX_ENV=dev mix build --zip
+
 release: build.engine
 	cd apps/expert &&\
 		mix deps.get &&\
@@ -81,7 +84,7 @@ release: build.engine
 release.local: build.engine
 	cd apps/expert &&\
 		mix deps.get &&\
-		EXPERT_RELEASE_MODE=burrito BURRITO_TARGET=$(local_target) MIX_ENV=prod mix release expert --overwrite
+		EXPERT_RELEASE_MODE=burrito BURRITO_TARGET=$(local_target) MIX_ENV=dev mix release expert --overwrite
 
 release.plain: build.engine
 	cd apps/expert &&\

apps/engine/.gitignore

@@ -24,3 +24,6 @@ engine-*.tar
 
 # Temporary files, for example, from tests.
 /tmp/
+
+# Ignore the package zip built via `mix build --zip`.
+engine-*.zip

apps/engine/.tool-versions

@@ -0,0 +1 @@
+elixir 1.17.2

apps/engine/lib/mix/tasks/build.ex

@@ -1,22 +1,48 @@
 defmodule Mix.Tasks.Build do
   use Mix.Task
 
-  def run(_args) do
-    Mix.Task.run("compile", [])
+  @options [
+    strict: [
+      path: :string,
+      zip: :boolean
+    ]
+  ]
+
+  def run(args) do
+    {opts, _, _} = OptionParser.parse(args, @options)
+    default_path = Path.join(["_build", "package", "engine"])
+    package_root = Keyword.get(opts, :path, default_path)
 
-    namespaced_dir = "_build/#{Mix.env()}_ns"
+    Mix.Task.run("compile", [])
 
     # Remove the existing namespaced dir
-    File.rm_rf(namespaced_dir)
+    File.rm_rf(package_root)
     # Create our namespaced area
-    File.mkdir_p(namespaced_dir)
+    File.mkdir_p(package_root)
 
     # Move our build artifacts from safekeeping to the build area
-    File.cp_r!("_build/#{Mix.env()}", namespaced_dir)
+    File.cp_r!("_build/#{Mix.env()}", package_root)
 
     # Namespace the new code
-    Mix.Task.run(:namespace, [
-      namespaced_dir
-    ])
+    Mix.Task.run(:namespace, [package_root])
+
+    if Keyword.get(opts, :zip, false) do
+      zip(package_root)
+      File.rm_rf(package_root)
+    end
+  end
+
+  defp zip(package_root) do
+    package_name = Path.basename(package_root)
+    versions = Forge.VM.Versions.current()
+    zip_output = Path.join(File.cwd!(), "engine-#{versions.elixir}-otp-#{versions.erlang}.zip")
+
+    package_root
+    |> Path.dirname()
+    |> File.cd!(fn ->
+      System.cmd("zip", ["-r", zip_output, package_name])
+    end)
+
+    IO.puts("Engine package created at: #{zip_output}")
   end
 end

apps/expert/lib/expert.ex

@@ -24,6 +24,10 @@ defmodule Expert do
 
   @dialyzer {:nowarn_function, apply_to_state: 2}
 
+  @expert_vsn Expert.MixProject.project()[:version]
+
+  def vsn, do: @expert_vsn
+
   def get_lsp, do: :persistent_term.get(:expert_lsp, nil)
 
   def start_link(args) do

apps/expert/lib/expert/engine_node.ex

@@ -115,7 +115,7 @@ defmodule Expert.EngineNode do
     bootstrap_args = [project, Document.Store.entropy(), all_app_configs()]
 
     with {:ok, node_pid} <- EngineSupervisor.start_project_node(project),
-         :ok <- start_node(project, glob_paths()),
+         :ok <- start_node(project, glob_paths(project)),
          :ok <- :rpc.call(node_name, Engine.Bootstrap, :init, bootstrap_args),
          :ok <- ensure_apps_started(node_name) do
       {:ok, node_name, node_pid}
@@ -152,7 +152,7 @@ defmodule Expert.EngineNode do
       ["/**/priv" | app_globs]
     end
 
-    def glob_paths do
+    def glob_paths(_) do
       for entry <- :code.get_path(),
           entry_string = List.to_string(entry),
           entry_string != ".",
@@ -165,9 +165,87 @@ defmodule Expert.EngineNode do
     # separately and copied to expert's priv directory.
     # When Engine is built in CI for a version matrix, we'll need to check if
     # we have the right version downloaded, and if not, we should download it.
-    defp glob_paths do
+    defp glob_paths(%Project{} = project) do
+      {:ok, elixir, _} = Expert.Port.elixir_executable(project)
+
+      {output, _} = System.cmd(elixir, ["-e", get_versions_code()])
+
+      case output |> String.trim() |> String.split("-") do
+        [elixir_vsn, erlang_vsn] ->
+          Logger.info(
+            "Detected project Elixir version: #{elixir_vsn}, Erlang version: #{erlang_vsn}"
+          )
+
+          find_engine_build(elixir_vsn, erlang_vsn)
+
+        _ ->
+          Logger.warning(
+            "Failed to detect Elixir and Erlang versions, using default Engine build"
+          )
+
+          default_glob_paths()
+      end
+    end
+
+    defp get_versions_code() do
+      ~S"""
+      major = :otp_release |> :erlang.system_info() |> List.to_string()
+      version_file = Path.join([:code.root_dir(), "releases", major, "OTP_VERSION"])
+
+      erlang_vsn =
+        try do
+          {:ok, contents} = File.read(version_file)
+          String.split(contents, "\n", trim: true)
+        else
+          [full] -> full
+          _ -> major
+        catch
+          :error ->
+            major
+        end
+
+      IO.puts("#{System.version()}-#{erlang_vsn}")
+      """
+    end
+
+    defp find_engine_build(elixir_vsn, erlang_vsn) do
+      user_data_path = :filename.basedir(:user_data, "Expert", %{version: Expert.vsn()})
+      engine_path = Path.join([user_data_path, "engine", "*-otp-*"])
+
+      Logger.info("Looking for Engine build at: #{engine_path}")
+
+      candidates =
+        for entry <- Path.wildcard(engine_path),
+            versions = entry |> Path.basename(entry) |> String.split("-"),
+            match?([_, "otp", _], versions),
+            [elixir, "otp", erlang] = versions,
+            Version.match?(elixir, "~> #{elixir_vsn}"),
+            Version.match?(erlang, "~> #{erlang_vsn}") do
+          entry
+        end
+
+      case List.first(candidates) do
+        nil ->
+          Logger.warning(
+            "No matching Engine build found for Elixir #{elixir_vsn} and Erlang #{erlang_vsn}"
+          )
+
+          default_glob_paths()
+
+        engine_path ->
+          Logger.info("Using Engine build at: #{engine_path}")
+          ebin_paths(engine_path)
+      end
+    end
+
+    defp default_glob_paths do
       :expert
       |> :code.priv_dir()
+      |> ebin_paths()
+    end
+
+    defp ebin_paths(base_path) do
+      base_path
       |> Path.join("lib/**/ebin")
       |> Path.wildcard()
     end

apps/expert/lib/expert/release.ex

@@ -4,7 +4,7 @@ defmodule Expert.Release do
 
     engine_path = Path.expand("../../../engine", __DIR__)
 
-    source = Path.join([engine_path, "_build/dev_ns"])
+    source = Path.join([engine_path, "_build/package/engine"])
 
     dest =
       Path.join([