Purge changed deps (#777)

* update readme

* run formatter

* Unload changed deps

Purge all modules from changed and removed deps, remove from code paths
Fixes #120
Fixes #688

* run ci on ubuntu 20.04

22.04 is OTP  >= 24

* fix warnings

Author: lukaszsamson

.github/workflows/ci.yml

@@ -11,7 +11,7 @@ on:
 jobs:
   mix_test:
     name: mix test (Elixir ${{matrix.elixir}} | Erlang/OTP ${{matrix.otp}})
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     strategy:
       fail-fast: false
       matrix:
@@ -53,7 +53,7 @@ jobs:
 
   static_analysis:
     name: static analysis (Elixir ${{matrix.elixir}} | Erlang/OTP ${{matrix.otp}})
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     strategy:
       matrix:
         include:

.github/workflows/docsite.yml

@@ -8,7 +8,7 @@ on:
 jobs:
   build:
     name: Build Mkdocs website
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     strategy:
         max-parallel: 1
     container:
@@ -27,7 +27,7 @@ jobs:
   publish:
     needs: build
     name: Publish Mkdocs website to GH Pages
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     strategy:
       max-parallel: 1
     steps:

.github/workflows/release-asset.yml

@@ -8,7 +8,7 @@ on:
 jobs:
   release:
     name: Create draft release
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     outputs:
       upload_url: ${{steps.create_release.outputs.upload_url}}
 
@@ -26,7 +26,7 @@ jobs:
 
   build:
     name: Build and publish release asset
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     needs: release
 
     strategy:

README.md

@@ -88,7 +88,7 @@ For VSCode install the extension: https://marketplace.visualstudio.com/items?ite
 
 Elixir:
 
-- 1.11.0 minimum
+- 1.12.3 minimum
 
 Erlang:
 

apps/language_server/lib/language_server/build.ex

@@ -14,23 +14,38 @@ defmodule ElixirLS.LanguageServer.Build do
             :timer.tc(fn ->
               Logger.info("Starting build with MIX_ENV: #{Mix.env()} MIX_TARGET: #{Mix.target()}")
 
+              # read cache before cleaning up mix state in reload_project
+              cached_deps = read_cached_deps()
+
               case reload_project() do
                 {:ok, mixfile_diagnostics} ->
                   # FIXME: Private API
-                  if Keyword.get(opts, :fetch_deps?) and
-                       Mix.Dep.load_on_environment([]) != cached_deps() do
-                    # NOTE: Clear deps cache when deps in mix.exs has change to prevent
-                    # formatter crash from clearing deps during build.
-                    :ok = Mix.Project.clear_deps_cache()
-                    fetch_deps()
-                  end
 
-                  # if we won't do it elixir >= 1.11 warns that protocols have already been consolidated
-                  purge_consolidated_protocols()
-                  {status, diagnostics} = run_mix_compile()
+                  try do
+                    # this call can raise
+                    current_deps = Mix.Dep.load_on_environment([])
+
+                    purge_changed_deps(current_deps, cached_deps)
+
+                    if Keyword.get(opts, :fetch_deps?) and current_deps != cached_deps do
+                      fetch_deps(current_deps)
+                    end
+
+                    # if we won't do it elixir >= 1.11 warns that protocols have already been consolidated
+                    purge_consolidated_protocols()
+                    {status, diagnostics} = run_mix_compile()
+
+                    diagnostics = Diagnostics.normalize(diagnostics, root_path)
+                    Server.build_finished(parent, {status, mixfile_diagnostics ++ diagnostics})
+                  rescue
+                    e ->
+                      Logger.warn(
+                        "Mix.Dep.load_on_environment([]) failed: #{inspect(e.__struct__)} #{Exception.message(e)}"
+                      )
 
-                  diagnostics = Diagnostics.normalize(diagnostics, root_path)
-                  Server.build_finished(parent, {status, mixfile_diagnostics ++ diagnostics})
+                      # TODO pass diagnostic
+                      Server.build_finished(parent, {:error, []})
+                  end
 
                 {:error, mixfile_diagnostics} ->
                   Server.build_finished(parent, {:error, mixfile_diagnostics})
@@ -73,8 +88,8 @@ defmodule ElixirLS.LanguageServer.Build do
       # see https://github.com/elixir-lsp/elixir-ls/issues/120
       # originally reported in https://github.com/JakeBecker/elixir-ls/issues/71
       # Note that `Mix.State.clear_cache()` is not enough (at least on elixir 1.14)
-      # FIXME: Private API
-      Mix.Dep.clear_cached()
+      Mix.Project.clear_deps_cache()
+      Mix.State.clear_cache()
 
       Mix.Task.clear()
 
@@ -179,24 +194,104 @@ defmodule ElixirLS.LanguageServer.Build do
     :code.delete(module)
   end
 
-  defp cached_deps do
-    try do
-      # FIXME: Private API
-      Mix.Dep.cached()
-    rescue
-      _ ->
-        []
+  defp purge_app(app) do
+    # TODO use hack with ets
+    modules =
+      case :application.get_key(app, :modules) do
+        {:ok, modules} -> modules
+        _ -> []
+      end
+
+    if modules != [] do
+      Logger.debug("Purging #{length(modules)} modules from #{app}")
+      for module <- modules, do: purge_module(module)
+    end
+
+    Logger.debug("Unloading #{app}")
+
+    case Application.stop(app) do
+      :ok -> :ok
+      {:error, :not_started} -> :ok
+      {:error, error} -> Logger.error("Application.stop failed for #{app}: #{inspect(error)}")
+    end
+
+    case Application.unload(app) do
+      :ok -> :ok
+      {:error, error} -> Logger.error("Application.unload failed for #{app}: #{inspect(error)}")
+    end
+
+    # Code.delete_path()
+  end
+
+  defp get_deps_by_app(deps), do: get_deps_by_app(deps, %{})
+  defp get_deps_by_app([], acc), do: acc
+
+  defp get_deps_by_app([curr = %Mix.Dep{app: app, deps: deps} | rest], acc) do
+    acc = get_deps_by_app(deps, acc)
+
+    list =
+      case acc[app] do
+        nil -> [curr]
+        list -> [curr | list]
+      end
+
+    get_deps_by_app(rest, acc |> Map.put(app, list))
+  end
+
+  defp maybe_purge_dep(%Mix.Dep{status: status, deps: deps} = dep) do
+    for dep <- deps, do: maybe_purge_dep(dep)
+
+    purge? =
+      case status do
+        {:nomatchvsn, _} -> true
+        :lockoutdated -> true
+        {:lockmismatch, _} -> true
+        _ -> false
+      end
+
+    if purge? do
+      purge_dep(dep)
+    end
+  end
+
+  defp purge_dep(%Mix.Dep{app: app} = dep) do
+    for path <- Mix.Dep.load_paths(dep) do
+      Code.delete_path(path)
+    end
+
+    purge_app(app)
+  end
+
+  defp purge_changed_deps(_current_deps, nil), do: :ok
+
+  defp purge_changed_deps(current_deps, cached_deps) do
+    current_deps_by_app = get_deps_by_app(current_deps)
+    cached_deps_by_app = get_deps_by_app(cached_deps)
+    removed_apps = Map.keys(cached_deps_by_app) -- Map.keys(current_deps_by_app)
+
+    removed_deps = cached_deps_by_app |> Map.take(removed_apps)
+
+    for {_app, deps} <- removed_deps,
+        dep <- deps do
+      purge_dep(dep)
+    end
+
+    for dep <- current_deps do
+      maybe_purge_dep(dep)
     end
   end
 
-  defp fetch_deps do
-    # FIXME: Private API and struct
+  defp fetch_deps(current_deps) do
+    # FIXME: private struct
     missing_deps =
-      Mix.Dep.load_on_environment([])
-      |> Enum.filter(fn %Mix.Dep{status: status} ->
+      current_deps
+      |> Enum.filter(fn %Mix.Dep{status: status, scm: scm} ->
         case status do
-          {:unavailable, _} -> true
+          {:unavailable, _} -> scm.fetchable?()
           {:nomatchvsn, _} -> true
+          :nolock -> true
+          :lockoutdated -> true
+          {:lockmismatch, _} -> true
           _ -> false
         end
       end)
@@ -215,6 +310,8 @@ defmodule ElixirLS.LanguageServer.Build do
         :info,
         "Done fetching deps"
       )
+    else
+      Logger.debug("All deps are up to date")
     end
 
     :ok
@@ -237,4 +334,17 @@ defmodule ElixirLS.LanguageServer.Build do
 
     Code.compiler_options(options)
   end
+
+  defp read_cached_deps() do
+    # FIXME: Private api
+    # we cannot use Mix.Dep.cached() here as it tries to load deps
+    if project = Mix.Project.get() do
+      env_target = {Mix.env(), Mix.target()}
+
+      case Mix.State.read_cache({:cached_deps, project}) do
+        {^env_target, deps} -> deps
+        _ -> nil
+      end
+    end
+  end
 end

apps/language_server/test/providers/completion_test.exs

@@ -141,7 +141,9 @@ defmodule ElixirLS.LanguageServer.Providers.CompletionTest do
 
     {line, char} = {3, 17}
     TestUtils.assert_has_cursor_char(text, line, char)
-    {:ok, %{"items" => [first_suggestion | _tail]}} = Completion.completion(text, line, char, @supports)
+
+    {:ok, %{"items" => [first_suggestion | _tail]}} =
+      Completion.completion(text, line, char, @supports)
 
     assert first_suggestion["label"] === "fn"
   end