Optimize macro traversals

Author: josevalim

lib/elixir/lib/module/types.ex

@@ -6,7 +6,8 @@ defmodule Module.Types do
   @no_infer [__protocol__: 1, behaviour_info: 1]
 
   @doc false
-  def infer(module, file, defs, private, used, env) do
+  def infer(module, file, defs, private, defmacrop, env) do
+    defmacrop = Map.from_keys(defmacrop, [])
     finder = &:lists.keyfind(&1, 1, defs)
 
     handler = fn meta, fun_arity, stack, context ->
@@ -29,23 +30,38 @@ defmodule Module.Types do
     context = context(%{})
 
     {types, %{local_sigs: local_sigs}} =
-      for {fun_arity, kind, meta, _clauses} = def <- defs,
-          kind == :def or kind == :defmacro,
-          reduce: {[], context} do
+      for {fun_arity, kind, meta, clauses} = def <- defs, reduce: {[], context} do
         {types, context} ->
-          {_kind, inferred, context} =
-            local_handler(meta, fun_arity, stack, context, fn _ -> def end)
-
-          if kind == :def and fun_arity not in @no_infer do
-            {[{fun_arity, inferred} | types], context}
-          else
-            {types, context}
+          cond do
+            kind in [:def, :defmacro] ->
+              {_kind, inferred, context} =
+                local_handler(meta, fun_arity, stack, context, fn _ -> def end)
+
+              if kind == :def and fun_arity not in @no_infer do
+                {[{fun_arity, inferred} | types], context}
+              else
+                {types, context}
+              end
+
+            kind == :defmacrop and is_map_key(defmacrop, fun_arity) ->
+              # Bypass the caching structure for defmacrop, that's because
+              # we don't need them stored in the signatures when we perform
+              # unreachable checks. This may cause defmacrop to be traversed
+              # twice if it uses default arguments (which is the only way
+              # to refer to another defmacrop in definitions).
+              {_kind, _inferred, context} =
+                local_handler(fun_arity, kind, meta, clauses, stack, context)
+
+              {types, context}
+
+            true ->
+              {types, context}
           end
       end
 
     unreachable =
       for {fun_arity, _kind, _meta, _defaults} = info <- private,
-          warn_unused_def(info, local_sigs, used, env),
+          warn_unused_def(info, local_sigs, defmacrop, env),
           not is_map_key(local_sigs, fun_arity),
           do: fun_arity
 
@@ -63,7 +79,7 @@ defmodule Module.Types do
   end
 
   defp warn_unused_def({fun_arity, kind, meta, 0}, reachable, used, env) do
-    case meta == false or Map.has_key?(reachable, fun_arity) or fun_arity in used do
+    case is_map_key(reachable, fun_arity) or is_map_key(used, fun_arity) do
       true -> :ok
       false -> :elixir_errors.file_warn(meta, env, __MODULE__, {:unused_def, fun_arity, kind})
     end
@@ -89,7 +105,7 @@ defmodule Module.Types do
   defp min_reachable_default(max, min, last, name, reachable, used) when max >= min do
     fun_arity = {name, max}
 
-    case Map.has_key?(reachable, fun_arity) or fun_arity in used do
+    case is_map_key(reachable, fun_arity) or is_map_key(used, fun_arity) do
       true -> min_reachable_default(max - 1, min, max, name, reachable, used)
       false -> min_reachable_default(max - 1, min, last, name, reachable, used)
     end
@@ -164,7 +180,7 @@ defmodule Module.Types do
 
   defp local_handler({fun, arity} = fun_arity, kind, meta, clauses, stack, context) do
     expected = List.duplicate(Descr.dynamic(), arity)
-    stack = stack |> fresh_stack(fun_arity) |> with_file_meta(meta)
+    stack = stack |> fresh_stack(kind, fun_arity) |> with_file_meta(meta)
 
     {_, _, mapping, clauses_types, clauses_context} =
       Enum.reduce(clauses, {0, 0, [], [], context}, fn
@@ -243,7 +259,7 @@ defmodule Module.Types do
 
   @doc false
   def stack(mode, file, module, function, no_warn_undefined, cache, handler)
-      when mode in [:static, :dynamic, :infer] do
+      when mode in [:static, :dynamic, :infer, :traversal] do
     %{
       # The fallback meta used for literals in patterns and guards
       meta: [],
@@ -275,6 +291,9 @@ defmodule Module.Types do
       #
       #   * :infer - Same as :dynamic but skips remote calls.
       #
+      #   * :traversal - Focused mostly on traversing AST, skips most type system
+      #     operations. Used by macros and functions which already have signatures.
+      #
       # The mode may also control exhaustiveness checks in the future (to be decided).
       # We may also want for applications with subtyping in dynamic mode to always
       # intersect with dynamic, but this mode may be too lax (to be decided based on
@@ -303,8 +322,15 @@ defmodule Module.Types do
     }
   end
 
-  defp fresh_stack(stack, function) do
-    %{stack | function: function}
+  defp fresh_stack(%{mode: mode} = stack, kind, function) do
+    mode =
+      cond do
+        kind in [:defmacro, :defmacrop] and mode == :infer -> :traversal
+        kind in [:def, :defp] and mode == :traversal -> :infer
+        true -> mode
+      end
+
+    %{stack | function: function, mode: mode}
   end
 
   defp fresh_context(context) do

lib/elixir/lib/module/types/apply.ex

@@ -2,7 +2,11 @@ defmodule Module.Types.Apply do
   # Typing functionality shared between Expr and Pattern.
   # Generic AST and Enum helpers go to Module.Types.Helpers.
   @moduledoc false
-  @max_clauses 32
+
+  # We limit the size of the union for two reasons:
+  # To avoid really large outputs in reports and to
+  # reduce the computation cost of inferred code.
+  @max_clauses 16
 
   alias Module.ParallelChecker
   import Module.Types.{Helpers, Descr}
@@ -188,7 +192,6 @@ defmodule Module.Types.Apply do
         {:erlang, :list_to_float, [{[non_empty_list(integer())], float()}]},
         {:erlang, :list_to_integer, [{[non_empty_list(integer())], integer()}]},
         {:erlang, :list_to_integer, [{[non_empty_list(integer()), integer()], integer()}]},
-        {:erlang, :list_to_tuple, [{[list(term())], dynamic(open_tuple([]))}]},
         {:erlang, :make_ref, [{[], reference()}]},
         {:erlang, :map_size, [{[open_map()], integer()}]},
         {:erlang, :node, [{[], atom()}]},
@@ -217,6 +220,7 @@ defmodule Module.Types.Apply do
         {:erlang, :element, [{[integer(), open_tuple([])], dynamic()}]},
         {:erlang, :insert_element,
          [{[integer(), open_tuple([]), term()], dynamic(open_tuple([]))}]},
+        {:erlang, :list_to_tuple, [{[list(term())], dynamic(open_tuple([]))}]},
         {:erlang, :max, [{[term(), term()], dynamic()}]},
         {:erlang, :min, [{[term(), term()], dynamic()}]},
         {:erlang, :orelse, [{[boolean(), term()], dynamic()}]},
@@ -256,6 +260,10 @@ defmodule Module.Types.Apply do
 
   Used only by info functions.
   """
+  def remote(_name, _args_types, _expr, %{mode: :traversal}, context) do
+    {dynamic(), context}
+  end
+
   def remote(name, args_types, expr, stack, context) do
     arity = length(args_types)
 
@@ -268,6 +276,10 @@ defmodule Module.Types.Apply do
   @doc """
   Applies a function in a given module.
   """
+  def remote(_module, _fun, _args_types, _expr, %{mode: :traversal}, context) do
+    {dynamic(), context}
+  end
+
   def remote(:erlang, :element, [_, tuple], {_, meta, [index, _]} = expr, stack, context)
       when is_integer(index) do
     case tuple_fetch(tuple, index - 1) do
@@ -515,11 +527,14 @@ defmodule Module.Types.Apply do
       false ->
         {dynamic(), context}
 
+      {_kind, _info, context} when stack.mode == :traversal ->
+        {dynamic(), context}
+
       {kind, info, context} ->
         case apply_signature(info, args_types, stack) do
           {:ok, indexes, type} ->
             context =
-              if stack != :infer and kind == :defp do
+              if stack.mode != :infer and kind == :defp do
                 update_in(context.local_used[fun_arity], fn current ->
                   if info == :none do
                     []
@@ -554,10 +569,13 @@ defmodule Module.Types.Apply do
 
     case stack.local_handler.(meta, fun_arity, stack, context) do
       false ->
+        {dynamic(fun()), context}
+
+      {_kind, _info, context} when stack.mode == :traversal ->
         {fun(), context}
 
       {kind, _info, context} ->
-        if stack != :infer and kind == :defp do
+        if stack.mode != :infer and kind == :defp do
           # Mark all clauses as used, as the function is being exported.
           {fun(), put_in(context.local_used[fun_arity], [])}
         else

lib/elixir/lib/module/types/expr.ex

@@ -221,7 +221,7 @@ defmodule Module.Types.Expr do
         {head_type, context} = of_expr(head, stack, context)
 
         context =
-          if stack.mode == :infer do
+          if stack.mode in [:infer, :traversal] do
             context
           else
             case truthness(head_type) do
@@ -386,14 +386,19 @@ defmodule Module.Types.Expr do
       )
       when is_atom(name) and is_integer(arity) do
     {remote_type, context} = of_expr(remote, stack, context)
+
     # TODO: We cannot return the unions of functions. Do we forbid this?
     # Do we check it is always the same return type? Do we simply say it is a function?
     {mods, context} = Of.modules(remote_type, name, arity, expr, meta, stack, context)
 
     context =
-      Enum.reduce(mods, context, &(Apply.signature(&1, name, arity, meta, stack, &2) |> elem(1)))
+      Enum.reduce(
+        mods,
+        context,
+        &(Apply.signature(&1, name, arity, meta, stack, &2) |> elem(1))
+      )
 
-    {fun(), context}
+    {dynamic(fun()), context}
   end
 
   # TODO: &foo/1
@@ -416,8 +421,12 @@ defmodule Module.Types.Expr do
   end
 
   # var
-  def of_expr(var, _stack, context) when is_var(var) do
-    {Of.var(var, context), context}
+  def of_expr(var, stack, context) when is_var(var) do
+    if stack.mode == :traversal do
+      {dynamic(), context}
+    else
+      {Of.var(var, context), context}
+    end
   end
 
   ## Try
@@ -551,15 +560,21 @@ defmodule Module.Types.Expr do
   defp dynamic_unless_static({_, _} = output, %{mode: :static}), do: output
   defp dynamic_unless_static({type, context}, %{mode: _}), do: {dynamic(type), context}
 
-  defp of_clauses(clauses, expected, info, stack, {acc, context}) do
+  defp of_clauses(clauses, expected, info, %{mode: mode} = stack, {acc, context}) do
     %{failed: failed?} = context
 
     Enum.reduce(clauses, {acc, context}, fn {:->, meta, [head, body]}, {acc, context} ->
       {failed?, context} = reset_failed(context, failed?)
       {patterns, guards} = extract_head(head)
       {_types, context} = Pattern.of_head(patterns, guards, expected, info, meta, stack, context)
       {body, context} = of_expr(body, stack, context)
-      {union(acc, body), set_failed(context, failed?)}
+      context = set_failed(context, failed?)
+
+      if mode == :traversal do
+        {dynamic(), context}
+      else
+        {union(acc, body), context}
+      end
     end)
   end
 

lib/elixir/lib/module/types/pattern.ex

@@ -25,6 +25,10 @@ defmodule Module.Types.Pattern do
      is refined, we restart at step 2.
 
   """
+  def of_head(_patterns, _guards, expected, _tag, _meta, %{mode: :traversal}, context) do
+    {expected, context}
+  end
+
   def of_head(patterns, guards, expected, tag, meta, stack, context) do
     stack = %{stack | meta: meta}
 
@@ -98,7 +102,13 @@ defmodule Module.Types.Pattern do
   This version tracks the whole expression in tracing,
   instead of only the pattern.
   """
-  def of_match(pattern, guards \\ [], expected, expr, tag, stack, context) do
+  def of_match(pattern, guards \\ [], expected, expr, tag, stack, context)
+
+  def of_match(_pattern, _guards, expected, _expr, _tag, %{mode: :traversal}, context) do
+    {expected, context}
+  end
+
+  def of_match(pattern, guards, expected, expr, tag, stack, context) do
     context = init_pattern_info(context)
     {tree, context} = of_pattern(pattern, [{:arg, 0, expr}], stack, context)
 

lib/elixir/src/elixir_def.erl

@@ -37,11 +37,10 @@ fun_for(Meta, Module, Name, Arity, Kinds, External) ->
     {[{_, Kind, LocalMeta, _, _, _}], ClausesPairs} ->
       case (Kinds == all) orelse (lists:member(Kind, Kinds)) of
         true ->
-          TrackLocal = (Kind == defmacrop),
-          TrackLocal andalso track_local(Module, Tuple),
-          Local = {value, fun(Fun, Args) -> invoke_local(Meta, Module, Fun, Args, TrackLocal, External) end},
+          (Kind == defmacrop) andalso track_defmacrop(Module, Tuple),
+          Local = {value, fun(Fun, Args) -> invoke_local(Meta, Module, Fun, Args, External) end},
           Clauses = [Clause || {_, Clause} <- ClausesPairs],
-          {Kind, elixir_erl:definition_to_anonymous(Kind, LocalMeta, Clauses, Local, External)};
+          elixir_erl:definition_to_anonymous(Kind, LocalMeta, Clauses, Local, External);
         false ->
           false
       end;
@@ -51,21 +50,20 @@ fun_for(Meta, Module, Name, Arity, Kinds, External) ->
     _:_ -> false
   end.
 
-invoke_local(Meta, Module, ErlName, Args, TrackLocal, External) ->
+invoke_local(Meta, Module, ErlName, Args, External) ->
   {Name, Arity} = elixir_utils:erl_fa_to_elixir_fa(ErlName, length(Args)),
 
   case fun_for(Meta, Module, Name, Arity, all, External) of
     false ->
       {current_stacktrace, [_ | T]} = erlang:process_info(self(), current_stacktrace),
       erlang:raise(error, undef, [{Module, Name, Arity, []} | T]);
-    {Kind, Fun} ->
-      (TrackLocal and (Kind == defp)) andalso track_local(Module, {Name, Arity}),
+    Fun ->
       apply(Fun, Args)
   end.
 
-track_local(Module, FunArity) ->
+track_defmacrop(Module, FunArity) ->
   {_, Bag} = elixir_module:data_tables(Module),
-  ets:insert(Bag, {macro_private_calls, FunArity}).
+  ets:insert(Bag, {defmacrop_calls, FunArity}).
 
 invoke_external(Meta, Mod, Name, Args, E) ->
   is_map(E) andalso elixir_env:trace({require, Meta, Mod, []}, E),

lib/elixir/src/elixir_dispatch.erl

@@ -170,20 +170,20 @@ expand_import(Meta, Name, Arity, E, Extra, AllowLocals, Trace) ->
     _ ->
       Local = AllowLocals andalso elixir_def:local_for(Meta, Name, Arity, [defmacro, defmacrop], E),
 
-      case {Dispatch, Local} of
+      case Dispatch of
         %% There is a local and an import. This is a conflict unless
         %% the receiver is the same as module (happens on bootstrap).
-        {{_, Receiver}, {_, _}} when Receiver /= Module ->
+        {_, Receiver} when Local /= false, Receiver /= Module ->
           {conflict, Receiver};
 
         %% There is no local. Dispatch the import.
-        {_, false} ->
+        _ when Local == false ->
           do_expand_import(Dispatch, Meta, Name, Arity, Module, E, Trace);
 
         %% Dispatch to the local.
-        {_, {_Kind, Fun}} ->
+        _ ->
           Trace andalso elixir_env:trace({local_macro, Meta, Name, Arity}, E),
-          {macro, Module, expander_macro_fun(Meta, Fun, Module, Name, E)}
+          {macro, Module, expander_macro_fun(Meta, Local, Module, Name, E)}
       end
   end.
 

lib/elixir/src/elixir_map.erl

@@ -192,7 +192,7 @@ maybe_load_struct(Meta, Name, Assocs, E) ->
       false ->
         Name:'__struct__'(Assocs);
 
-      {_, ExternalFun} ->
+      ExternalFun ->
         %% There is an inherent race condition when using external_for.
         %% By the time we got to execute the function, the ETS table
         %% with temporary definitions for the given module may no longer

lib/elixir/src/elixir_module.erl

@@ -187,8 +187,8 @@ compile(Meta, Module, ModuleAsCharlist, Block, Vars, Prune, E) ->
           case elixir_config:is_bootstrap() of
             true -> {#{}, []};
             false ->
-              Used = bag_lookup_element(DataBag, macro_private_calls, 2),
-              'Elixir.Module.Types':infer(Module, File, AllDefinitions, NewPrivate, Used, E)
+              Defmacrop = bag_lookup_element(DataBag, defmacrop_calls, 2),
+              'Elixir.Module.Types':infer(Module, File, AllDefinitions, NewPrivate, Defmacrop, E)
           end,
 
         RawCompileOpts = bag_lookup_element(DataBag, {accumulate, compile}, 2),