Infer types and use them across remote calls (#13981)

This also unifies handling of deterministic builds with Erlang/OTP
as well as warnings_as_errors.

Author: josevalim

.github/workflows/ci.yml

@@ -18,16 +18,16 @@ permissions:
 
 jobs:
   test_linux:
-    name: Ubuntu 24.04, Erlang/OTP ${{ matrix.otp_version }}
+    name: Ubuntu 24.04, Erlang/OTP ${{ matrix.otp_version }}${{ matrix.deterministic && ' (deterministic)' || '' }}
     strategy:
       fail-fast: false
       matrix:
         include:
+          - otp_version: "27.1"
+            deterministic: true
           - otp_version: "27.1"
             otp_latest: true
-            erlc_opts: "warnings_as_errors"
           - otp_version: "27.0"
-            erlc_opts: "warnings_as_errors"
           - otp_version: "26.0"
           - otp_version: "25.3"
           - otp_version: "25.0"
@@ -36,18 +36,16 @@ jobs:
           - otp_version: maint
             development: true
     runs-on: ubuntu-24.04
-    # Earlier Erlang/OTP versions ignored compiler directives
-    # when using warnings_as_errors. So we only set ERLC_OPTS
-    # from Erlang/OTP 27+.
-    env:
-      ERLC_OPTS: ${{ matrix.erlc_opts || '' }}
     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 50
       - uses: erlef/setup-beam@v1
         with:
           otp-version: ${{ matrix.otp_version }}
+      - name: Set ERL_COMPILER_OPTIONS
+        if: ${{ matrix.deterministic }}
+        run: echo "ERL_COMPILER_OPTIONS=deterministic" >> $GITHUB_ENV
       - name: Compile Elixir
         run: |
           make compile
@@ -72,12 +70,12 @@ jobs:
           cd ../elixir/
           make docs
       - name: Check reproducible builds
+        if: ${{ matrix.deterministic }}
         run: |
           rm -rf .git
           # Recompile System without .git
           cd lib/elixir && ../../bin/elixirc -o ebin lib/system.ex && cd -
           taskset 1 make check_reproducible
-        if: ${{ matrix.otp_latest }}
 
   test_windows:
     name: Windows Server 2019, Erlang/OTP ${{ matrix.otp_version }}

CHANGELOG.md

@@ -83,6 +83,7 @@ You may also prefer to write using guards:
   * [Kernel] Track the type of tuples in patterns and inside `elem/2`
   * [Kernel] Perform validation of root AST nodes in `unquote` and `unquote_splicing` to catch bugs earlier
   * [Kernel] Add source, behaviour, and record information to Docs chunk metadata
+  * [Kernel] Support deterministic builds in tandem with Erlang by setting `ERL_COMPILER_OPTIONS=deterministic`. Keep in mind deterministic builds strip source and other compile time information, which may be relevant for programs
   * [List] Add `List.ends_with?/2`
   * [Macro] Improve `dbg` handling of `if/2`, `with/1` and of code blocks
   * [Macro] Add `Macro.struct_info!/2` to return struct information mirroring `mod.__info__(:struct)`
@@ -156,6 +157,7 @@ You may also prefer to write using guards:
 
 #### Elixir
 
+  * [Code] Setting `:warnings_as_errors` is deprecated via `Code.put_compiler_option/2`. This must not affect developers, as the `:warnings_as_errors` option is managed by Mix tasks, and not directly used via the `Code` module
   * [Enumerable] Deprecate returning a two-arity function in `Enumerable.slice/1`
   * [List] `List.zip/1` is deprecated in favor of `Enum.zip/1`
   * [Module] Deprecate `Module.eval_quoted/3` in favor of `Code.eval_quoted/3`

Makefile

@@ -5,7 +5,7 @@ MAN_PREFIX ?= $(SHARE_PREFIX)/man
 CANONICAL := main/
 ELIXIRC := bin/elixirc --ignore-module-conflict $(ELIXIRC_OPTS)
 ERLC := erlc -I lib/elixir/include
-ERL_MAKE := if [ -n "$(ERLC_OPTS)" ]; then ERL_COMPILER_OPTIONS=$(ERLC_OPTS) erl -make; else erl -make; fi
+ERL_MAKE := erl -make
 ERL := erl -I lib/elixir/include -noshell -pa lib/elixir/ebin
 GENERATE_APP := $(CURDIR)/lib/elixir/scripts/generate_app.escript
 VERSION := $(strip $(shell cat VERSION))

README.md

@@ -103,7 +103,9 @@ on Windows](https://github.com/elixir-lang/elixir/wiki/Windows).
 In case you want to use this Elixir version as your system version,
 you need to add the `bin` directory to [your PATH environment variable](https://elixir-lang.org/install.html#setting-path-environment-variable).
 
-Additionally, you may choose to run the test suite with `make clean test`.
+When updating the repository, you may want to run `make clean` before
+recompiling. For deterministic builds, you should set the environment
+variable `ERL_COMPILER_OPTIONS=deterministic`.
 
 ## Contributing
 

lib/elixir/Emakefile

@@ -9,8 +9,8 @@
   warn_deprecated_function,
   warn_obsolete_guard,
   warn_exported_vars,
-  %% warn_missing_spec,
-  %% warn_untyped_record,
+  %% Enable this when we require Erlang/OTP 27+
+  %% warnings_as_errors,
   debug_info,
   {outdir, "ebin/"}
 ]}.

lib/elixir/lib/code.ex

@@ -249,8 +249,8 @@ defmodule Code do
     :debug_info,
     :ignore_already_consolidated,
     :ignore_module_conflict,
-    :relative_paths,
-    :warnings_as_errors
+    :infer_signatures,
+    :relative_paths
   ]
 
   @list_compiler_options [:no_warn_undefined, :tracers, :parser_options]
@@ -1562,8 +1562,8 @@ defmodule Code do
 
   ## Examples
 
-      Code.compiler_options(warnings_as_errors: true)
-      #=> %{warnings_as_errors: false}
+      Code.compiler_options(infer_signatures: false)
+      #=> %{infer_signatures: true}
 
   """
   @spec compiler_options(Enumerable.t({atom, term})) :: %{optional(atom) => term}
@@ -1592,6 +1592,12 @@ defmodule Code do
     :elixir_config.get(key)
   end
 
+  # TODO: Remove me in Elixir v2.0
+  def get_compiler_option(:warnings_as_errors) do
+    IO.warn(":warnings_as_errors is deprecated as part of Code.get_compiler_option/1")
+    :ok
+  end
+
   @doc """
   Returns a list with all available compiler options.
 
@@ -1620,16 +1626,19 @@ defmodule Code do
       Defaults to `true`.
 
     * `:debug_info` - when `true`, retains debug information in the compiled
-      module. Defaults to `true`.
-      This enables static analysis tools as it allows developers to
-      partially reconstruct the original source code. Therefore, disabling
+      module. This option can also be overridden per module using the `@compile`
+      directive. Defaults to `true`.
+
+      This enables tooling to partially reconstruct the original source code,
+      for instance, to perform static analysis of code. Therefore, disabling
       `:debug_info` is not recommended as it removes the ability of the
       Elixir compiler and other tools to provide feedback. If you want to
       remove the `:debug_info` while deploying, tools like `mix release`
       already do such by default.
-      Additionally, `mix test` disables it via the `:test_elixirc_options`
-      project configuration option.
-      This option can also be overridden per module using the `@compile` directive.
+
+      Other environments, such as `mix test`, automatically disables this
+      via the `:test_elixirc_options` project configuration, as there is
+      typically no need to store debug chunks for test files.
 
     * `:ignore_already_consolidated` (since v1.10.0) - when `true`, does not warn
       when a protocol has already been consolidated and a new implementation is added.
@@ -1638,13 +1647,19 @@ defmodule Code do
     * `:ignore_module_conflict` - when `true`, does not warn when a module has
       already been defined. Defaults to `false`.
 
+    * `:infer_signatures` (since v1.18.0) - when `false`, it disables module-local
+      signature inference used when type checking remote calls to the compiled
+      module. Type checking will be executed regardless of this value of this option.
+      Defaults to `true`.
+
+      `mix test` automatically disables this option via the `:test_elixirc_options`
+      project configuration, as there is typically no need to store infer signatures
+      for test files.
+
     * `:relative_paths` - when `true`, uses relative paths in quoted nodes,
       warnings, and errors generated by the compiler. Note disabling this option
       won't affect runtime warnings and errors. Defaults to `true`.
 
-    * `:warnings_as_errors` - causes compilation to fail when warnings are
-      generated. Defaults to `false`.
-
     * `:no_warn_undefined` (since v1.10.0) - list of modules and `{Mod, fun, arity}`
       tuples that will not emit warnings that the module or function does not exist
       at compilation time. Pass atom `:all` to skip warning for all undefined
@@ -1690,6 +1705,16 @@ defmodule Code do
     :ok
   end
 
+  # TODO: Remove me in Elixir v2.0
+  def put_compiler_option(:warnings_as_errors, _value) do
+    IO.warn(
+      ":warnings_as_errors is deprecated as part of Code.put_compiler_option/2, " <>
+        "pass it as option to Kernel.ParallelCompiler instead"
+    )
+
+    :ok
+  end
+
   def put_compiler_option(:no_warn_undefined, value) do
     if value != :all and not is_list(value) do
       raise "compiler option :no_warn_undefined should be a list or the atom :all, " <>
@@ -1719,7 +1744,7 @@ defmodule Code do
     :ok
   end
 
-  # TODO: Make this option have no effect on Elixir v2.0
+  # TODO: Remove this option on Elixir v2.0
   # TODO: Warn if mode is :warn on Elixir v1.19
   def put_compiler_option(:on_undefined_variable, value) when value in [:raise, :warn] do
     :elixir_config.put(:on_undefined_variable, value)

lib/elixir/lib/kernel/cli.ex

@@ -9,6 +9,7 @@ defmodule Kernel.CLI do
     compile: [],
     no_halt: false,
     compiler_options: [],
+    warnings_as_errors: false,
     errors: [],
     verbose_compile: false,
     profile: nil,
@@ -315,8 +316,7 @@ defmodule Kernel.CLI do
   end
 
   defp parse_argv([~c"--warnings-as-errors" | t], %{mode: :elixirc} = config) do
-    compiler_options = [{:warnings_as_errors, true} | config.compiler_options]
-    parse_argv(t, %{config | compiler_options: compiler_options})
+    parse_argv(t, %{config | warnings_as_errors: true})
   end
 
   defp parse_argv([~c"--verbose" | t], %{mode: :elixirc} = config) do
@@ -499,15 +499,11 @@ defmodule Kernel.CLI do
               ]
             end
 
-          profile_opts =
-            if config.profile do
-              [profile: config.profile]
-            else
-              []
-            end
-
           output = IO.chardata_to_string(config.output)
-          opts = verbose_opts ++ profile_opts
+
+          opts =
+            verbose_opts ++
+              [profile: config.profile, warnings_as_errors: config.warnings_as_errors]
 
           case Kernel.ParallelCompiler.compile_to_path(files, output, opts) do
             {:ok, _, _} -> :ok

lib/elixir/lib/kernel/parallel_compiler.ex

@@ -258,7 +258,7 @@ defmodule Kernel.ParallelCompiler do
     {status, modules_or_errors, info} =
       try do
         outcome = spawn_workers(schedulers, checker, files, output, options)
-        {outcome, Code.get_compiler_option(:warnings_as_errors)}
+        {outcome, Keyword.get(options, :warnings_as_errors, false)}
       else
         {{:ok, _, %{runtime_warnings: r_warnings, compile_warnings: c_warnings} = info}, true}
         when r_warnings != [] or c_warnings != [] ->

lib/elixir/lib/macro.ex

@@ -854,7 +854,7 @@ defmodule Macro do
   @spec struct_info!(module(), Macro.Env.t()) ::
           [%{field: atom(), required: boolean(), default: term()}]
   def struct_info!(module, env) when is_atom(module) do
-    case :elixir_map.maybe_load_struct_info([line: env.line], module, [], env) do
+    case :elixir_map.maybe_load_struct_info([line: env.line], module, [], true, env) do
       {:ok, info} -> info
       {:error, desc} -> raise ArgumentError, List.to_string(:elixir_map.format_error(desc))
     end

lib/elixir/lib/module/parallel_checker.ex

@@ -191,7 +191,8 @@ defmodule Module.ParallelChecker do
   or if the function does not exist return `{:error, :function}`.
   """
   @spec fetch_export(cache(), module(), atom(), arity()) ::
-          {:ok, mode(), kind(), binary() | nil} | {:error, :function | :module}
+          {:ok, mode(), binary() | nil, {:infer, [term()]} | :none}
+          | {:error, :function | :module}
   def fetch_export({server, ets}, module, fun, arity) do
     case :ets.lookup(ets, module) do
       [] ->
@@ -203,7 +204,7 @@ defmodule Module.ParallelChecker do
 
       [{_key, mode}] ->
         case :ets.lookup(ets, {module, {fun, arity}}) do
-          [{_key, reason}] -> {:ok, mode, reason}
+          [{_key, reason, signature}] -> {:ok, mode, reason, signature}
           [] -> {:error, :function}
         end
     end
@@ -369,13 +370,13 @@ defmodule Module.ParallelChecker do
             true ->
               {mode, exports} = info_exports(module)
               deprecated = info_deprecated(module)
-              cache_info(ets, module, exports, deprecated, mode)
+              cache_info(ets, module, exports, deprecated, %{}, mode)
 
             false ->
               # Or load exports from chunk
               with {^module, binary, _filename} <- object_code,
                    {:ok, {^module, [exports: exports]}} <- :beam_lib.chunks(binary, [:exports]) do
-                cache_info(ets, module, exports, %{}, :erlang)
+                cache_info(ets, module, exports, %{}, %{}, :erlang)
               else
                 _ ->
                   :ets.insert(ets, {module, false})
@@ -417,25 +418,28 @@ defmodule Module.ParallelChecker do
         behaviour_exports(map) ++
         for({function, :def, _meta, _clauses} <- map.definitions, do: function)
 
-    deprecated = Map.new(map.deprecated)
-    cache_info(ets, map.module, exports, deprecated, :elixir)
+    cache_info(ets, map.module, exports, Map.new(map.deprecated), map.signatures, :elixir)
   end
 
-  defp cache_info(ets, module, exports, deprecated, mode) do
-    Enum.each(exports, fn {fun, arity} ->
-      reason = Map.get(deprecated, {fun, arity})
-      :ets.insert(ets, {{module, {fun, arity}}, reason})
+  defp cache_info(ets, module, exports, deprecated, sigs, mode) do
+    Enum.each(exports, fn fa ->
+      reason = Map.get(deprecated, fa)
+      :ets.insert(ets, {{module, fa}, reason, Map.get(sigs, fa, :none)})
     end)
 
     :ets.insert(ets, {module, mode})
   end
 
   defp cache_chunk(ets, module, exports) do
     Enum.each(exports, fn {{fun, arity}, info} ->
-      :ets.insert(ets, {{module, {fun, arity}}, Map.get(info, :deprecated)})
+      # TODO: Match on signature directly in Elixir v1.22+
+      :ets.insert(
+        ets,
+        {{module, {fun, arity}}, Map.get(info, :deprecated), Map.get(info, :sig, :none)}
+      )
     end)
 
-    :ets.insert(ets, {{module, {:__info__, 1}}, nil})
+    :ets.insert(ets, {{module, {:__info__, 1}}, nil, :none})
     :ets.insert(ets, {module, :elixir})
   end