Remove dependency on manifest files

Author: José Valim

lib/mix/lib/mix/compilers/elixir.ex

@@ -0,0 +1,208 @@
+defmodule Mix.Compilers.Elixir do
+  @moduledoc false
+
+  @doc """
+  Compiles stale Elixir files.
+
+  It expects a manifest file, the source directories, the extensions
+  to read in sources, the destination directory, a flag to know if
+  compilation is being forced or not and a callback to be invoked
+  once (and only if) compilation starts.
+
+  The manifest is written down with information including dependencies
+  in between modules, which helps it recompile only the modules that
+  have changed at runtime.
+  """
+  def compile(manifest, srcs, exts, dest, force, on_start) do
+    all = Mix.Utils.extract_files(srcs, exts)
+    all_entries = read_manifest(manifest)
+
+    removed =
+      for {_b, _m, source, _d, _f} <- all_entries, not(source in all), do: source
+
+    changed =
+      if force do
+        # A config, path dependency or manifest has
+        # changed, let's just compile everything
+        all
+      else
+        modified = Mix.Utils.last_modified(manifest)
+
+        # Otherwise let's start with the new ones
+        # plus the ones that have changed
+        for(source <- all,
+            not Enum.any?(all_entries, fn {_b, _m, s, _d, _f} -> s == source end),
+            do: source)
+          ++
+        for({_b, _m, source, _d, files} <- all_entries,
+            Mix.Utils.stale?([source|files], [modified]),
+            do: source)
+      end
+
+    {entries, changed} = remove_stale_entries(all_entries, removed ++ changed, [], [])
+    stale = changed -- removed
+
+    cond do
+      stale != [] ->
+        do_compile(manifest, entries, stale, dest, on_start)
+        :ok
+      removed != [] ->
+        :ok
+      true ->
+        :noop
+    end
+  end
+
+  @doc """
+  Removes compiled files.
+  """
+  def clean(manifest) do
+    case File.read(manifest) do
+      {:ok, contents} ->
+        contents
+        |> String.split("\n")
+        |> Enum.each(& &1 |> String.split("\t") |> hd |> File.rm)
+        File.rm(manifest)
+      {:error, _} ->
+        :ok
+    end
+  end
+
+  defp do_compile(manifest, entries, stale, dest, on_start) do
+    Mix.Project.build_structure
+    on_start.()
+    cwd = File.cwd!
+
+    # Starts a server responsible for keeping track which files
+    # were compiled and the dependencies in between them.
+    {:ok, pid} = Agent.start_link(fn ->
+      Enum.map(entries, &Tuple.insert_at(&1, 5, nil))
+    end)
+
+    try do
+      Kernel.ParallelCompiler.files :lists.usort(stale),
+        each_module: &each_module(pid, dest, cwd, &1, &2, &3),
+        each_file: &each_file(&1)
+      Agent.cast pid, fn entries ->
+        write_manifest(manifest, entries)
+        entries
+      end
+    after
+      Agent.stop pid
+    end
+
+    :ok
+  end
+
+  defp each_module(pid, dest, cwd, source, module, binary) do
+    source = Path.relative_to(source, cwd)
+    bin    = Atom.to_string(module)
+    beam   = dest
+             |> Path.join(bin <> ".beam")
+             |> Path.relative_to(cwd)
+
+    deps = Kernel.LexicalTracker.remotes(module)
+           |> List.delete(module)
+           |> :lists.usort
+           |> Enum.map(&Atom.to_string(&1))
+           |> Enum.reject(&match?("elixir_" <> _, &1))
+
+    files = get_beam_files(binary, cwd)
+            |> List.delete(source)
+            |> Enum.filter(&(Path.type(&1) == :relative))
+
+    Agent.cast pid, &:lists.keystore(beam, 1, &1, {beam, bin, source, deps, files, binary})
+  end
+
+  defp get_beam_files(binary, cwd) do
+    case :beam_lib.chunks(binary, [:abstract_code]) do
+      {:ok, {_, [abstract_code: {:raw_abstract_v1, code}]}} ->
+        for {:attribute, _, :file, {file, _}} <- code,
+            File.exists?(file) do
+          Path.relative_to(file, cwd)
+        end
+      _ ->
+        []
+    end
+  end
+
+  defp each_file(file) do
+    Mix.shell.info "Compiled #{file}"
+  end
+
+  ## Resolution
+
+  # This function receives the manifest entries and some source
+  # files that have changed. It then, recursively, figures out
+  # all the files that changed (thanks to the dependencies) and
+  # return their sources as the remaining entries.
+  defp remove_stale_entries(all, []) do
+    {all, []}
+  end
+
+  defp remove_stale_entries(all, changed) do
+    remove_stale_entries(all, :lists.usort(changed), [], [])
+  end
+
+  defp remove_stale_entries([{beam, module, source, _d, _f} = entry|t], changed, removed, acc) do
+    if source in changed do
+      File.rm(beam)
+      remove_stale_entries(t, changed, [module|removed], acc)
+    else
+      remove_stale_entries(t, changed, removed, [entry|acc])
+    end
+  end
+
+  defp remove_stale_entries([], changed, removed, acc) do
+    # If any of the dependencies for the remaining entries
+    # were removed, get its source so we can remove them.
+    next_changed = for {_b, _m, source, deps, _f} <- acc,
+                    Enum.any?(deps, &(&1 in removed)),
+                    do: source
+
+    {acc, next} = remove_stale_entries(Enum.reverse(acc), next_changed)
+    {acc, next ++ changed}
+  end
+
+  ## Manifest handling
+
+  # Reads the manifest returning the results as tuples.
+  # The beam files are read, removed and stored in memory.
+  defp read_manifest(manifest) do
+    case File.read(manifest) do
+      {:ok, contents} ->
+        Enum.reduce String.split(contents, "\n"), [], fn x, acc ->
+          case String.split(x, "\t") do
+            [beam, module, source|deps] ->
+              {deps, files} =
+                case Enum.split_while(deps, &(&1 != "Elixir")) do
+                  {deps, ["Elixir"|files]} -> {deps, files}
+                  {deps, _} -> {deps, []}
+                end
+              [{beam, module, source, deps, files}|acc]
+            _ ->
+              acc
+          end
+        end
+      {:error, _} ->
+        []
+    end
+  end
+
+  # Writes the manifest separating entries by tabs.
+  defp write_manifest(_manifest, []) do
+    :ok
+  end
+
+  defp write_manifest(manifest, entries) do
+    lines = Enum.map(entries, fn
+      {beam, module, source, deps, files, binary} ->
+        if binary, do: File.write!(beam, binary)
+        tail = deps ++ ["Elixir"] ++ files
+        [beam, module, source | tail] |> Enum.join("\t")
+    end)
+
+    File.mkdir_p!(Path.dirname(manifest))
+    File.write!(manifest, Enum.join(lines, "\n"))
+  end
+end

lib/mix/lib/mix/compilers/erlang.ex

@@ -0,0 +1,144 @@
+defmodule Mix.Compilers.Erlang do
+  @moduledoc false
+
+  @doc """
+  Compiles the files in `src_dirs` with given extensions into
+  the destination, automatically invoking the callback for each
+  stale input and output pair (or for all if `force` is true) and
+  removing files that no longer have a source, while keeping the
+  manifest up to date.
+
+  ## Examples
+
+  For example, a simple compiler for Lisp Flavored Erlang
+  would be implemented like:
+
+      manifest = Path.join Mix.Project.manifest_path, ".compile.lfe"
+      dest = Mix.Project.compile_path
+
+      compile manifest, [{"src", dest}], :lfe, :beam, opts[:force], fn
+        input, output ->
+          :lfe_comp.file(to_erl_file(input),
+                         [output_dir: Path.dirname(output)])
+      end
+
+  The command above will:
+
+  1. Look for files ending with the `lfe` extension in `src`
+     and their `beam` counterpart in `ebin`;
+  2. For each stale file (or for all if `force` is true),
+     invoke the callback passing the calculated input
+     and output;
+  3. Update the manifest with the newly compiled outputs;
+  4. Remove any output in the manifest that that does not
+     have an equivalent source;
+
+  The callback must return `{:ok, mod}` or `:error` in case
+  of error. An error is raised at the end if any of the
+  files failed to compile.
+  """
+  def compile(manifest, mappings, src_ext, dest_ext, force, callback) do
+    files = for {src, dest} <- mappings do
+              extract_targets(src, src_ext, dest, dest_ext, force)
+            end |> Enum.concat
+    compile(manifest, files, callback)
+  end
+
+  @doc """
+  Compiles the given src/dest tuples.
+
+  A manifest file and a callback to be invoked for each src/dest pair
+  must be given. A src/dest pair where destination is nil is considered
+  to be up to date and won't be (re-)compiled.
+  """
+  def compile(manifest, tuples, callback) do
+    stale = for {:stale, src, dest} <- tuples, do: {src, dest}
+
+    # Get the previous entries from the manifest
+    entries = read_manifest(manifest)
+
+    # Files to remove are the ones in the manifest
+    # but they no longer have a source
+    removed = Enum.filter(entries, fn entry ->
+      not Enum.any?(tuples, fn {_status, _src, dest} -> dest == entry end)
+    end)
+
+    if stale == [] && removed == [] do
+      :noop
+    else
+      # Build the project structure so we can write down compiled files.
+      Mix.Project.build_structure
+
+      # Remove manifest entries with no source
+      Enum.each(removed, &File.rm/1)
+
+      # Compile stale files and print the results
+      results = for {input, output} <- stale do
+        interpret_result(input, callback.(input, output))
+      end
+
+      # Write final entries to manifest
+      entries = (entries -- removed) ++ Enum.map(stale, &elem(&1, 1))
+      write_manifest(manifest, :lists.usort(entries))
+
+      # Raise if any error, return :ok otherwise
+      if :error in results, do: raise CompileError
+      :ok
+    end
+  end
+
+  @doc """
+  Removes compiled files.
+  """
+  def clean(manifest) do
+    Enum.each read_manifest(manifest), &File.rm/1
+    File.rm manifest
+  end
+
+  @doc """
+  Converts the given file to a format accepted by
+  the Erlang compilation tools.
+  """
+  def to_erl_file(file) do
+    to_char_list(file)
+  end
+
+  defp extract_targets(src_dir, src_ext, dest_dir, dest_ext, force) do
+    files = Mix.Utils.extract_files(List.wrap(src_dir), List.wrap(src_ext))
+
+    for file <- files do
+      module = module_from_artifact(file)
+      target = Path.join(dest_dir, module <> "." <> to_string(dest_ext))
+
+      if force || Mix.Utils.stale?([file], [target]) do
+        {:stale, file, target}
+      else
+        {:ok, file, target}
+      end
+    end
+  end
+
+  defp module_from_artifact(artifact) do
+    artifact |> Path.basename |> Path.rootname
+  end
+
+  defp interpret_result(file, result) do
+    case result do
+      {:ok, _} -> Mix.shell.info "Compiled #{file}"
+      :error -> :error
+    end
+    result
+  end
+
+  defp read_manifest(file) do
+    case File.read(file) do
+      {:ok, contents} -> String.split(contents, "\n")
+      {:error, _} -> []
+    end
+  end
+
+  defp write_manifest(file, entries) do
+    Path.dirname(file) |> File.mkdir_p!
+    File.write!(file, Enum.join(entries, "\n"))
+  end
+end

lib/mix/lib/mix/tasks/clean.ex

@@ -20,12 +20,12 @@ defmodule Mix.Tasks.Clean do
   def run(args) do
     {opts, _, _} = OptionParser.parse(args)
 
-    manifests = Mix.Tasks.Compile.manifests
-    Enum.each(manifests, fn(manifest) ->
-      Enum.each Mix.Utils.read_manifest(manifest),
-                &(&1 |> String.split("\t") |> hd |> File.rm)
-      File.rm(manifest)
-    end)
+    for compiler <- Mix.Tasks.Compile.compilers() do
+      module = Mix.Task.get!("compile.#{compiler}")
+      if function_exported?(module, :clean, 0) do
+        module.clean
+      end
+    end
 
     if opts[:all] do
       Mix.Task.run("deps.clean", args)